/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */#include"jit/Ion.h"#include"mozilla/IntegerPrintfMacros.h"#include"mozilla/MemoryReporting.h"#include"mozilla/SizePrintfMacros.h"#include"mozilla/ThreadLocal.h"#include"jscompartment.h"#include"jsgc.h"#include"jsprf.h"#include"gc/Marking.h"#include"jit/AliasAnalysis.h"#include"jit/AlignmentMaskAnalysis.h"#include"jit/BacktrackingAllocator.h"#include"jit/BaselineFrame.h"#include"jit/BaselineInspector.h"#include"jit/BaselineJIT.h"#include"jit/CacheIRSpewer.h"#include"jit/CodeGenerator.h"#include"jit/EagerSimdUnbox.h"#include"jit/EdgeCaseAnalysis.h"#include"jit/EffectiveAddressAnalysis.h"#include"jit/FlowAliasAnalysis.h"#include"jit/FoldLinearArithConstants.h"#include"jit/InstructionReordering.h"#include"jit/IonAnalysis.h"#include"jit/IonBuilder.h"#include"jit/IonIC.h"#include"jit/IonOptimizationLevels.h"#include"jit/JitcodeMap.h"#include"jit/JitCommon.h"#include"jit/JitCompartment.h"#include"jit/JitSpewer.h"#include"jit/LICM.h"#include"jit/LIR.h"#include"jit/LoopUnroller.h"#include"jit/Lowering.h"#include"jit/PerfSpewer.h"#include"jit/RangeAnalysis.h"#include"jit/ScalarReplacement.h"#include"jit/Sink.h"#include"jit/StupidAllocator.h"#include"jit/ValueNumbering.h"#include"jit/WasmBCE.h"#include"vm/Debugger.h"#include"vm/HelperThreads.h"#include"vm/TraceLogging.h"#include"vtune/VTuneWrapper.h"#include"jscompartmentinlines.h"#include"jsobjinlines.h"#include"jsscriptinlines.h"#include"jit/JitFrames-inl.h"#include"jit/shared/Lowering-shared-inl.h"#include"vm/Debugger-inl.h"#include"vm/EnvironmentObject-inl.h"#include"vm/Stack-inl.h"usingnamespacejs;usingnamespacejs::jit;// Assert that JitCode is gc::Cell aligned.JS_STATIC_ASSERT(sizeof(JitCode)%gc::CellAlignBytes==0);staticMOZ_THREAD_LOCAL(JitContext*)TlsJitContext;staticJitContext*CurrentJitContext(){if(!TlsJitContext.init())returnnullptr;returnTlsJitContext.get();}voidjit::SetJitContext(JitContext*ctx){TlsJitContext.set(ctx);}JitContext*jit::GetJitContext(){MOZ_ASSERT(CurrentJitContext());returnCurrentJitContext();}JitContext*jit::MaybeGetJitContext(){returnCurrentJitContext();}JitContext::JitContext(JSContext*cx,TempAllocator*temp):cx(cx),temp(temp),runtime(CompileRuntime::get(cx->runtime())),compartment(CompileCompartment::get(cx->compartment())),prev_(CurrentJitContext()),assemblerCount_(0){SetJitContext(this);}JitContext::JitContext(CompileRuntime*rt,CompileCompartment*comp,TempAllocator*temp):cx(nullptr),temp(temp),runtime(rt),compartment(comp),prev_(CurrentJitContext()),assemblerCount_(0){SetJitContext(this);}JitContext::JitContext(CompileRuntime*rt):cx(nullptr),temp(nullptr),runtime(rt),compartment(nullptr),prev_(CurrentJitContext()),assemblerCount_(0){SetJitContext(this);}JitContext::JitContext(TempAllocator*temp):cx(nullptr),temp(temp),runtime(nullptr),compartment(nullptr),prev_(CurrentJitContext()),assemblerCount_(0){SetJitContext(this);}JitContext::JitContext(CompileRuntime*rt,TempAllocator*temp):cx(nullptr),temp(temp),runtime(rt),compartment(nullptr),prev_(CurrentJitContext()),assemblerCount_(0){SetJitContext(this);}JitContext::JitContext():cx(nullptr),temp(nullptr),runtime(nullptr),compartment(nullptr),prev_(CurrentJitContext()),assemblerCount_(0){SetJitContext(this);}JitContext::~JitContext(){SetJitContext(prev_);}booljit::InitializeIon(){if(!TlsJitContext.init())returnfalse;CheckLogging();#ifdef JS_CACHEIR_SPEWconstchar*env=getenv("CACHEIR_LOGS");if(env&&env[0])CacheIRSpewer::singleton().init();#endif#if defined(JS_CODEGEN_ARM)InitARMFlags();#endifCheckPerf();returntrue;}JitRuntime::JitRuntime(JSRuntime*rt):execAlloc_(rt),backedgeExecAlloc_(rt),exceptionTail_(nullptr),bailoutTail_(nullptr),profilerExitFrameTail_(nullptr),enterJIT_(nullptr),bailoutHandler_(nullptr),argumentsRectifier_(nullptr),argumentsRectifierReturnAddr_(nullptr),invalidator_(nullptr),debugTrapHandler_(nullptr),baselineDebugModeOSRHandler_(nullptr),functionWrappers_(nullptr),preventBackedgePatching_(false),jitcodeGlobalTable_(nullptr){}JitRuntime::~JitRuntime(){js_delete(functionWrappers_.ref());// By this point, the jitcode global table should be empty.MOZ_ASSERT_IF(jitcodeGlobalTable_,jitcodeGlobalTable_->empty());js_delete(jitcodeGlobalTable_.ref());}boolJitRuntime::initialize(JSContext*cx,AutoLockForExclusiveAccess&lock){AutoAtomsCompartmentac(cx,lock);JitContextjctx(cx,nullptr);if(!cx->compartment()->ensureJitCompartmentExists(cx))returnfalse;functionWrappers_=cx->new_<VMWrapperMap>(cx);if(!functionWrappers_||!functionWrappers_->init())returnfalse;JitSpew(JitSpew_Codegen,"# Emitting profiler exit frame tail stub");profilerExitFrameTail_=generateProfilerExitFrameTailStub(cx);if(!profilerExitFrameTail_)returnfalse;JitSpew(JitSpew_Codegen,"# Emitting exception tail stub");void*handler=JS_FUNC_TO_DATA_PTR(void*,jit::HandleException);exceptionTail_=generateExceptionTailStub(cx,handler);if(!exceptionTail_)returnfalse;JitSpew(JitSpew_Codegen,"# Emitting bailout tail stub");bailoutTail_=generateBailoutTailStub(cx);if(!bailoutTail_)returnfalse;if(cx->runtime()->jitSupportsFloatingPoint){JitSpew(JitSpew_Codegen,"# Emitting bailout tables");// Initialize some Ion-only stubs that require floating-point support.BailoutTableVector&bailoutTables=bailoutTables_.writeRef();if(!bailoutTables.reserve(FrameSizeClass::ClassLimit().classId()))returnfalse;for(uint32_tid=0;;id++){FrameSizeClassclass_=FrameSizeClass::FromClass(id);if(class_==FrameSizeClass::ClassLimit())break;bailoutTables.infallibleAppend((JitCode*)nullptr);JitSpew(JitSpew_Codegen,"# Bailout table");bailoutTables[id]=generateBailoutTable(cx,id);if(!bailoutTables[id])returnfalse;}JitSpew(JitSpew_Codegen,"# Emitting bailout handler");bailoutHandler_=generateBailoutHandler(cx);if(!bailoutHandler_)returnfalse;JitSpew(JitSpew_Codegen,"# Emitting invalidator");invalidator_=generateInvalidator(cx);if(!invalidator_)returnfalse;}JitSpew(JitSpew_Codegen,"# Emitting sequential arguments rectifier");argumentsRectifier_=generateArgumentsRectifier(cx,&argumentsRectifierReturnAddr_.writeRef());if(!argumentsRectifier_)returnfalse;JitSpew(JitSpew_Codegen,"# Emitting EnterJIT sequence");enterJIT_=generateEnterJIT(cx,EnterJitOptimized);if(!enterJIT_)returnfalse;JitSpew(JitSpew_Codegen,"# Emitting EnterBaselineJIT sequence");enterBaselineJIT_=generateEnterJIT(cx,EnterJitBaseline);if(!enterBaselineJIT_)returnfalse;JitSpew(JitSpew_Codegen,"# Emitting Pre Barrier for Value");valuePreBarrier_=generatePreBarrier(cx,MIRType::Value);if(!valuePreBarrier_)returnfalse;JitSpew(JitSpew_Codegen,"# Emitting Pre Barrier for String");stringPreBarrier_=generatePreBarrier(cx,MIRType::String);if(!stringPreBarrier_)returnfalse;JitSpew(JitSpew_Codegen,"# Emitting Pre Barrier for Object");objectPreBarrier_=generatePreBarrier(cx,MIRType::Object);if(!objectPreBarrier_)returnfalse;JitSpew(JitSpew_Codegen,"# Emitting Pre Barrier for Shape");shapePreBarrier_=generatePreBarrier(cx,MIRType::Shape);if(!shapePreBarrier_)returnfalse;JitSpew(JitSpew_Codegen,"# Emitting Pre Barrier for ObjectGroup");objectGroupPreBarrier_=generatePreBarrier(cx,MIRType::ObjectGroup);if(!objectGroupPreBarrier_)returnfalse;JitSpew(JitSpew_Codegen,"# Emitting malloc stub");mallocStub_=generateMallocStub(cx);if(!mallocStub_)returnfalse;JitSpew(JitSpew_Codegen,"# Emitting free stub");freeStub_=generateFreeStub(cx);if(!freeStub_)returnfalse;JitSpew(JitSpew_Codegen,"# Emitting VM function wrappers");for(VMFunction*fun=VMFunction::functions;fun;fun=fun->next){JitSpew(JitSpew_Codegen,"# VM function wrapper");if(!generateVMWrapper(cx,*fun))returnfalse;}JitSpew(JitSpew_Codegen,"# Emitting lazy link stub");lazyLinkStub_=generateLazyLinkStub(cx);if(!lazyLinkStub_)returnfalse;jitcodeGlobalTable_=cx->new_<JitcodeGlobalTable>();if(!jitcodeGlobalTable_)returnfalse;returntrue;}JitCode*JitRuntime::debugTrapHandler(JSContext*cx){if(!debugTrapHandler_){// JitRuntime code stubs are shared across compartments and have to// be allocated in the atoms compartment.AutoLockForExclusiveAccesslock(cx);AutoAtomsCompartmentac(cx,lock);debugTrapHandler_=generateDebugTrapHandler(cx);}returndebugTrapHandler_;}uint8_t*JSContext::allocateOsrTempData(size_tsize){osrTempData_=(uint8_t*)js_realloc(osrTempData_,size);returnosrTempData_;}voidJSContext::freeOsrTempData(){js_free(osrTempData_);osrTempData_=nullptr;}voidJitZoneGroup::patchIonBackedges(JSContext*cx,BackedgeTargettarget){if(target==BackedgeLoopHeader){// We must be on the active thread. The caller must use// AutoPreventBackedgePatching to ensure we don't reenter.MOZ_ASSERT(cx->runtime()->jitRuntime()->preventBackedgePatching());MOZ_ASSERT(CurrentThreadCanAccessRuntime(cx->runtime()));}else{// We must be called from InterruptRunningJitCode, or a signal handler// triggered there. rt->handlingJitInterrupt() ensures we can't reenter// this code.MOZ_ASSERT(!cx->runtime()->jitRuntime()->preventBackedgePatching());MOZ_ASSERT(cx->handlingJitInterrupt());}// Do nothing if we know all backedges are already jumping to `target`.if(backedgeTarget_==target)return;backedgeTarget_=target;cx->runtime()->jitRuntime()->backedgeExecAlloc().makeAllWritable();// Patch all loop backedges in Ion code so that they either jump to the// normal loop header or to an interrupt handler each time they run.for(InlineListIterator<PatchableBackedge>iter(backedgeList().begin());iter!=backedgeList().end();iter++){PatchableBackedge*patchableBackedge=*iter;if(target==BackedgeLoopHeader)PatchBackedge(patchableBackedge->backedge,patchableBackedge->loopHeader,target);elsePatchBackedge(patchableBackedge->backedge,patchableBackedge->interruptCheck,target);}cx->runtime()->jitRuntime()->backedgeExecAlloc().makeAllExecutable();}JitZoneGroup::JitZoneGroup(ZoneGroup*group):backedgeTarget_(group,BackedgeLoopHeader),backedgeList_(group){}JitCompartment::JitCompartment():stubCodes_(nullptr),stringConcatStub_(nullptr),regExpMatcherStub_(nullptr),regExpSearcherStub_(nullptr),regExpTesterStub_(nullptr){}JitCompartment::~JitCompartment(){js_delete(stubCodes_);}boolJitCompartment::initialize(JSContext*cx){stubCodes_=cx->new_<ICStubCodeMap>(cx->runtime());if(!stubCodes_)returnfalse;if(!stubCodes_->init()){ReportOutOfMemory(cx);returnfalse;}returntrue;}boolJitCompartment::ensureIonStubsExist(JSContext*cx){if(!stringConcatStub_){stringConcatStub_=generateStringConcatStub(cx);if(!stringConcatStub_)returnfalse;}returntrue;}boolJitZone::init(JSContext*cx){if(!baselineCacheIRStubCodes_.init()){ReportOutOfMemory(cx);returnfalse;}returntrue;}voidjit::FreeIonBuilder(IonBuilder*builder){// The builder is allocated into its LifoAlloc, so destroying that will// destroy the builder and all other data accumulated during compilation,// except any final codegen (which includes an assembler and needs to be// explicitly destroyed).js_delete(builder->backgroundCodegen());js_delete(builder->alloc().lifoAlloc());}voidjit::FinishOffThreadBuilder(JSRuntime*runtime,IonBuilder*builder,constAutoLockHelperThreadState&locked){// Clean the references to the pending IonBuilder, if we just finished it.if(builder->script()->baselineScript()->hasPendingIonBuilder()&&builder->script()->baselineScript()->pendingIonBuilder()==builder){builder->script()->baselineScript()->removePendingIonBuilder(builder->script());}// If the builder is still in one of the helper thread list, then remove it.if(builder->isInList())builder->script()->zone()->group()->ionLazyLinkListRemove(builder);// Clear the recompiling flag of the old ionScript, since we continue to// use the old ionScript if recompiling fails.if(builder->script()->hasIonScript())builder->script()->ionScript()->clearRecompiling();// Clean up if compilation did not succeed.if(builder->script()->isIonCompilingOffThread()){IonScript*ion=nullptr;AbortReasonOr<Ok>status=builder->getOffThreadStatus();if(status.isErr()&&status.unwrapErr()==AbortReason::Disable)ion=ION_DISABLED_SCRIPT;builder->script()->setIonScript(runtime,ion);}// Free Ion LifoAlloc off-thread. Free on the main thread if this OOMs.if(!StartOffThreadIonFree(builder,locked))FreeIonBuilder(builder);}staticvoidMarkJitProfilerEvent(JSRuntime*rt,JSScript*script,constchar*event){if(rt->geckoProfiler().enabled()){// Register event with profiler.// Format of event payload string:// "<filename>:<lineno>"// Get the script filename, if any, and its length.constchar*filename=script->filename();if(filename==nullptr)filename="<unknown>";// Construct the descriptive string.UniqueCharsbuf=JS_smprintf("Invalidate %s:%"PRIuSIZE,filename,script->lineno());// Ignore the event on allocation failure.if(buf){rt->geckoProfiler().markEvent(buf.get());}}}staticboolLinkCodeGen(JSContext*cx,IonBuilder*builder,CodeGenerator*codegen){RootedScriptscript(cx,builder->script());TraceLoggerThread*logger=TraceLoggerForCurrentThread(cx);TraceLoggerEventevent(TraceLogger_AnnotateScripts,script);AutoTraceLoglogScript(logger,event);AutoTraceLoglogLink(logger,TraceLogger_IonLinking);if(!codegen->link(cx,builder->constraints()))returnfalse;MarkJitProfilerEvent(cx->runtime(),script,"Ion compilation finished");returntrue;}staticboolLinkBackgroundCodeGen(JSContext*cx,IonBuilder*builder){CodeGenerator*codegen=builder->backgroundCodegen();if(!codegen)returnfalse;JitContextjctx(cx,&builder->alloc());// Root the assembler until the builder is finished below. As it was// constructed off thread, the assembler has not been rooted previously,// though any GC activity would discard the builder.MacroAssembler::AutoRootermasm(cx,&codegen->masm);returnLinkCodeGen(cx,builder,codegen);}voidjit::LinkIonScript(JSContext*cx,HandleScriptcalleeScript){IonBuilder*builder;{AutoLockHelperThreadStatelock;// Get the pending builder from the Ion frame.MOZ_ASSERT(calleeScript->hasBaselineScript());builder=calleeScript->baselineScript()->pendingIonBuilder();calleeScript->baselineScript()->removePendingIonBuilder(calleeScript);// Remove from pending.cx->zone()->group()->ionLazyLinkListRemove(builder);}{AutoEnterAnalysisenterTypes(cx);if(!LinkBackgroundCodeGen(cx,builder)){// Silently ignore OOM during code generation. The assembly code// doesn't has code to handle it after linking happened. So it's// not OK to throw a catchable exception from there.cx->clearPendingException();// Reset the TypeZone's compiler output for this script, if any.InvalidateCompilerOutputsForScript(cx,calleeScript);}}{AutoLockHelperThreadStatelock;FinishOffThreadBuilder(cx->runtime(),builder,lock);}}uint8_t*jit::LazyLinkTopActivation(){// First frame should be an exit frame.JSContext*cx=TlsContext.get();JitFrameIteratorit(cx);LazyLinkExitFrameLayout*ll=it.exitFrame()->as<LazyLinkExitFrameLayout>();RootedScriptcalleeScript(cx,ScriptFromCalleeToken(ll->jsFrame()->calleeToken()));LinkIonScript(cx,calleeScript);MOZ_ASSERT(calleeScript->hasBaselineScript());MOZ_ASSERT(calleeScript->baselineOrIonRawPointer());returncalleeScript->baselineOrIonRawPointer();}/* static */voidJitRuntime::Trace(JSTracer*trc,AutoLockForExclusiveAccess&lock){MOZ_ASSERT(!JS::CurrentThreadIsHeapMinorCollecting());// Shared stubs are allocated in the atoms compartment, so do not iterate// them after the atoms heap after it has been "finished."if(trc->runtime()->atomsAreFinished())return;Zone*zone=trc->runtime()->atomsCompartment(lock)->zone();for(autoi=zone->cellIter<JitCode>();!i.done();i.next()){JitCode*code=i;TraceRoot(trc,&code,"wrapper");}}/* static */voidJitRuntime::TraceJitcodeGlobalTableForMinorGC(JSTracer*trc){if(trc->runtime()->geckoProfiler().enabled()&&trc->runtime()->hasJitRuntime()&&trc->runtime()->jitRuntime()->hasJitcodeGlobalTable()){trc->runtime()->jitRuntime()->getJitcodeGlobalTable()->traceForMinorGC(trc);}}/* static */boolJitRuntime::MarkJitcodeGlobalTableIteratively(GCMarker*marker){if(marker->runtime()->hasJitRuntime()&&marker->runtime()->jitRuntime()->hasJitcodeGlobalTable()){returnmarker->runtime()->jitRuntime()->getJitcodeGlobalTable()->markIteratively(marker);}returnfalse;}/* static */voidJitRuntime::SweepJitcodeGlobalTable(JSRuntime*rt){if(rt->hasJitRuntime()&&rt->jitRuntime()->hasJitcodeGlobalTable())rt->jitRuntime()->getJitcodeGlobalTable()->sweep(rt);}voidJitCompartment::sweep(FreeOp*fop,JSCompartment*compartment){// Any outstanding compilations should have been cancelled by the GC.MOZ_ASSERT(!HasOffThreadIonCompile(compartment));stubCodes_->sweep();// If the sweep removed a bailout Fallback stub, nullptr the corresponding return addr.for(auto&it:bailoutReturnStubInfo_){if(!stubCodes_->lookup(it.key))it=BailoutReturnStubInfo();}if(stringConcatStub_&&IsAboutToBeFinalizedUnbarriered(&stringConcatStub_))stringConcatStub_=nullptr;if(regExpMatcherStub_&&IsAboutToBeFinalizedUnbarriered(®ExpMatcherStub_))regExpMatcherStub_=nullptr;if(regExpSearcherStub_&&IsAboutToBeFinalizedUnbarriered(®ExpSearcherStub_))regExpSearcherStub_=nullptr;if(regExpTesterStub_&&IsAboutToBeFinalizedUnbarriered(®ExpTesterStub_))regExpTesterStub_=nullptr;for(ReadBarrieredObject&obj:simdTemplateObjects_){if(obj&&IsAboutToBeFinalized(&obj))obj.set(nullptr);}}voidJitZone::sweep(FreeOp*fop){baselineCacheIRStubCodes_.sweep();}size_tJitCompartment::sizeOfIncludingThis(mozilla::MallocSizeOfmallocSizeOf)const{size_tn=mallocSizeOf(this);if(stubCodes_)n+=stubCodes_->sizeOfIncludingThis(mallocSizeOf);returnn;}voidJitZone::addSizeOfIncludingThis(mozilla::MallocSizeOfmallocSizeOf,size_t*jitZone,size_t*baselineStubsOptimized,size_t*cachedCFG)const{*jitZone+=mallocSizeOf(this);*jitZone+=baselineCacheIRStubCodes_.sizeOfExcludingThis(mallocSizeOf);*jitZone+=ionCacheIRStubInfoSet_.sizeOfExcludingThis(mallocSizeOf);*baselineStubsOptimized+=optimizedStubSpace_.sizeOfExcludingThis(mallocSizeOf);*cachedCFG+=cfgSpace_.sizeOfExcludingThis(mallocSizeOf);}JitCode*JitRuntime::getBailoutTable(constFrameSizeClass&frameClass)const{MOZ_ASSERT(frameClass!=FrameSizeClass::None());returnbailoutTables_.ref()[frameClass.classId()];}JitCode*JitRuntime::getVMWrapper(constVMFunction&f)const{MOZ_ASSERT(functionWrappers_);MOZ_ASSERT(functionWrappers_->initialized());JitRuntime::VMWrapperMap::Ptrp=functionWrappers_->readonlyThreadsafeLookup(&f);MOZ_ASSERT(p);returnp->value();}template<AllowGCallowGC>JitCode*JitCode::New(JSContext*cx,uint8_t*code,uint32_tbufferSize,uint32_theaderSize,ExecutablePool*pool,CodeKindkind){JitCode*codeObj=Allocate<JitCode,allowGC>(cx);if(!codeObj){pool->release(headerSize+bufferSize,kind);returnnullptr;}new(codeObj)JitCode(code,bufferSize,headerSize,pool,kind);returncodeObj;}templateJitCode*JitCode::New<CanGC>(JSContext*cx,uint8_t*code,uint32_tbufferSize,uint32_theaderSize,ExecutablePool*pool,CodeKindkind);templateJitCode*JitCode::New<NoGC>(JSContext*cx,uint8_t*code,uint32_tbufferSize,uint32_theaderSize,ExecutablePool*pool,CodeKindkind);voidJitCode::copyFrom(MacroAssembler&masm){// Store the JitCode pointer right before the code buffer, so we can// recover the gcthing from relocation tables.*(JitCode**)(code_-sizeof(JitCode*))=this;insnSize_=masm.instructionsSize();masm.executableCopy(code_);jumpRelocTableBytes_=masm.jumpRelocationTableBytes();masm.copyJumpRelocationTable(code_+jumpRelocTableOffset());dataRelocTableBytes_=masm.dataRelocationTableBytes();masm.copyDataRelocationTable(code_+dataRelocTableOffset());masm.processCodeLabels(code_);}voidJitCode::traceChildren(JSTracer*trc){// Note that we cannot mark invalidated scripts, since we've basically// corrupted the code stream by injecting bailouts.if(invalidated())return;if(jumpRelocTableBytes_){uint8_t*start=code_+jumpRelocTableOffset();CompactBufferReaderreader(start,start+jumpRelocTableBytes_);MacroAssembler::TraceJumpRelocations(trc,this,reader);}if(dataRelocTableBytes_){// If we're moving objects, we need writable JIT code.boolmovingObjects=JS::CurrentThreadIsHeapMinorCollecting()||zone()->isGCCompacting();MaybeAutoWritableJitCodeawjc(this,movingObjects?Reprotect:DontReprotect);uint8_t*start=code_+dataRelocTableOffset();CompactBufferReaderreader(start,start+dataRelocTableBytes_);MacroAssembler::TraceDataRelocations(trc,this,reader);}}voidJitCode::finalize(FreeOp*fop){// If this jitcode had a bytecode map, it must have already been removed.#ifdef DEBUGJSRuntime*rt=fop->runtime();if(hasBytecodeMap_){MOZ_ASSERT(rt->jitRuntime()->hasJitcodeGlobalTable());MOZ_ASSERT(!rt->jitRuntime()->getJitcodeGlobalTable()->lookup(raw()));}#endif#ifdef MOZ_VTUNEvtune::UnmarkCode(this);#endifMOZ_ASSERT(pool_);// With W^X JIT code, reprotecting memory for each JitCode instance is// slow, so we record the ranges and poison them later all at once. It's// safe to ignore OOM here, it just means we won't poison the code.if(fop->appendJitPoisonRange(JitPoisonRange(pool_,code_-headerSize_,headerSize_+bufferSize_))){pool_->addRef();}code_=nullptr;// Code buffers are stored inside ExecutablePools. Pools are refcounted.// Releasing the pool may free it. Horrible hack: if we are using perf// integration, we don't want to reuse code addresses, so we just leak the// memory instead.if(!PerfEnabled())pool_->release(headerSize_+bufferSize_,CodeKind(kind_));pool_=nullptr;}IonScript::IonScript():method_(nullptr),deoptTable_(nullptr),osrPc_(nullptr),osrEntryOffset_(0),skipArgCheckEntryOffset_(0),invalidateEpilogueOffset_(0),invalidateEpilogueDataOffset_(0),numBailouts_(0),hasProfilingInstrumentation_(false),recompiling_(false),runtimeData_(0),runtimeSize_(0),icIndex_(0),icEntries_(0),safepointIndexOffset_(0),safepointIndexEntries_(0),safepointsStart_(0),safepointsSize_(0),frameSlots_(0),frameSize_(0),bailoutTable_(0),bailoutEntries_(0),osiIndexOffset_(0),osiIndexEntries_(0),snapshots_(0),snapshotsListSize_(0),snapshotsRVATableSize_(0),constantTable_(0),constantEntries_(0),backedgeList_(0),backedgeEntries_(0),invalidationCount_(0),recompileInfo_(),osrPcMismatchCounter_(0),fallbackStubSpace_(){}IonScript*IonScript::New(JSContext*cx,RecompileInforecompileInfo,uint32_tframeSlots,uint32_targumentSlots,uint32_tframeSize,size_tsnapshotsListSize,size_tsnapshotsRVATableSize,size_trecoversSize,size_tbailoutEntries,size_tconstants,size_tsafepointIndices,size_tosiIndices,size_ticEntries,size_truntimeSize,size_tsafepointsSize,size_tbackedgeEntries,size_tsharedStubEntries,OptimizationLeveloptimizationLevel){constexprsize_tDataAlignment=sizeof(void*);if(snapshotsListSize>=MAX_BUFFER_SIZE||(bailoutEntries>=MAX_BUFFER_SIZE/sizeof(uint32_t))){ReportOutOfMemory(cx);returnnullptr;}// This should not overflow on x86, because the memory is already allocated// *somewhere* and if their total overflowed there would be no memory left// at all.size_tpaddedSnapshotsSize=AlignBytes(snapshotsListSize+snapshotsRVATableSize,DataAlignment);size_tpaddedRecoversSize=AlignBytes(recoversSize,DataAlignment);size_tpaddedBailoutSize=AlignBytes(bailoutEntries*sizeof(uint32_t),DataAlignment);size_tpaddedConstantsSize=AlignBytes(constants*sizeof(Value),DataAlignment);size_tpaddedSafepointIndicesSize=AlignBytes(safepointIndices*sizeof(SafepointIndex),DataAlignment);size_tpaddedOsiIndicesSize=AlignBytes(osiIndices*sizeof(OsiIndex),DataAlignment);size_tpaddedICEntriesSize=AlignBytes(icEntries*sizeof(uint32_t),DataAlignment);size_tpaddedRuntimeSize=AlignBytes(runtimeSize,DataAlignment);size_tpaddedSafepointSize=AlignBytes(safepointsSize,DataAlignment);size_tpaddedBackedgeSize=AlignBytes(backedgeEntries*sizeof(PatchableBackedge),DataAlignment);size_tpaddedSharedStubSize=AlignBytes(sharedStubEntries*sizeof(IonICEntry),DataAlignment);size_tbytes=paddedSnapshotsSize+paddedRecoversSize+paddedBailoutSize+paddedConstantsSize+paddedSafepointIndicesSize+paddedOsiIndicesSize+paddedICEntriesSize+paddedRuntimeSize+paddedSafepointSize+paddedBackedgeSize+paddedSharedStubSize;IonScript*script=cx->zone()->pod_malloc_with_extra<IonScript,uint8_t>(bytes);if(!script)returnnullptr;new(script)IonScript();uint32_toffsetCursor=sizeof(IonScript);script->runtimeData_=offsetCursor;script->runtimeSize_=runtimeSize;offsetCursor+=paddedRuntimeSize;script->icIndex_=offsetCursor;script->icEntries_=icEntries;offsetCursor+=paddedICEntriesSize;script->safepointIndexOffset_=offsetCursor;script->safepointIndexEntries_=safepointIndices;offsetCursor+=paddedSafepointIndicesSize;script->safepointsStart_=offsetCursor;script->safepointsSize_=safepointsSize;offsetCursor+=paddedSafepointSize;script->bailoutTable_=offsetCursor;script->bailoutEntries_=bailoutEntries;offsetCursor+=paddedBailoutSize;script->osiIndexOffset_=offsetCursor;script->osiIndexEntries_=osiIndices;offsetCursor+=paddedOsiIndicesSize;script->snapshots_=offsetCursor;script->snapshotsListSize_=snapshotsListSize;script->snapshotsRVATableSize_=snapshotsRVATableSize;offsetCursor+=paddedSnapshotsSize;script->recovers_=offsetCursor;script->recoversSize_=recoversSize;offsetCursor+=paddedRecoversSize;script->constantTable_=offsetCursor;script->constantEntries_=constants;offsetCursor+=paddedConstantsSize;script->backedgeList_=offsetCursor;script->backedgeEntries_=backedgeEntries;offsetCursor+=paddedBackedgeSize;script->sharedStubList_=offsetCursor;script->sharedStubEntries_=sharedStubEntries;offsetCursor+=paddedSharedStubSize;script->frameSlots_=frameSlots;script->argumentSlots_=argumentSlots;script->frameSize_=frameSize;script->recompileInfo_=recompileInfo;script->optimizationLevel_=optimizationLevel;returnscript;}voidIonScript::adoptFallbackStubs(FallbackICStubSpace*stubSpace){fallbackStubSpace()->adoptFrom(stubSpace);}voidIonScript::trace(JSTracer*trc){if(method_)TraceEdge(trc,&method_,"method");if(deoptTable_)TraceEdge(trc,&deoptTable_,"deoptimizationTable");for(size_ti=0;i<numConstants();i++)TraceEdge(trc,&getConstant(i),"constant");// Mark all IC stub codes hanging off the IC stub entries.for(size_ti=0;i<numSharedStubs();i++){IonICEntry&ent=sharedStubList()[i];ent.trace(trc);}// Trace caches so that the JSScript pointer can be updated if moved.for(size_ti=0;i<numICs();i++)getICFromIndex(i).trace(trc);}/* static */voidIonScript::writeBarrierPre(Zone*zone,IonScript*ionScript){if(zone->needsIncrementalBarrier())ionScript->trace(zone->barrierTracer());}voidIonScript::copySnapshots(constSnapshotWriter*writer){MOZ_ASSERT(writer->listSize()==snapshotsListSize_);memcpy((uint8_t*)this+snapshots_,writer->listBuffer(),snapshotsListSize_);MOZ_ASSERT(snapshotsRVATableSize_);MOZ_ASSERT(writer->RVATableSize()==snapshotsRVATableSize_);memcpy((uint8_t*)this+snapshots_+snapshotsListSize_,writer->RVATableBuffer(),snapshotsRVATableSize_);}voidIonScript::copyRecovers(constRecoverWriter*writer){MOZ_ASSERT(writer->size()==recoversSize_);memcpy((uint8_t*)this+recovers_,writer->buffer(),recoversSize_);}voidIonScript::copySafepoints(constSafepointWriter*writer){MOZ_ASSERT(writer->size()==safepointsSize_);memcpy((uint8_t*)this+safepointsStart_,writer->buffer(),safepointsSize_);}voidIonScript::copyBailoutTable(constSnapshotOffset*table){memcpy(bailoutTable(),table,bailoutEntries_*sizeof(uint32_t));}voidIonScript::copyConstants(constValue*vp){for(size_ti=0;i<constantEntries_;i++)constants()[i].init(vp[i]);}voidIonScript::copyPatchableBackedges(JSContext*cx,JitCode*code,PatchableBackedgeInfo*backedges,MacroAssembler&masm){JitZoneGroup*jzg=cx->zone()->group()->jitZoneGroup;JitRuntime::AutoPreventBackedgePatchingapbp(cx->runtime());for(size_ti=0;i<backedgeEntries_;i++){PatchableBackedgeInfo&info=backedges[i];PatchableBackedge*patchableBackedge=&backedgeList()[i];info.backedge.fixup(&masm);CodeLocationJumpbackedge(code,info.backedge);CodeLocationLabelloopHeader(code,CodeOffset(info.loopHeader->offset()));CodeLocationLabelinterruptCheck(code,CodeOffset(info.interruptCheck->offset()));new(patchableBackedge)PatchableBackedge(backedge,loopHeader,interruptCheck);// Point the backedge to either of its possible targets, matching the// other backedges in the runtime.if(jzg->backedgeTarget()==JitZoneGroup::BackedgeInterruptCheck)PatchBackedge(backedge,interruptCheck,JitZoneGroup::BackedgeInterruptCheck);elsePatchBackedge(backedge,loopHeader,JitZoneGroup::BackedgeLoopHeader);jzg->addPatchableBackedge(cx->runtime()->jitRuntime(),patchableBackedge);}}voidIonScript::copySafepointIndices(constSafepointIndex*si,MacroAssembler&masm){// Jumps in the caches reflect the offset of those jumps in the compiled// code, not the absolute positions of the jumps. Update according to the// final code address now.SafepointIndex*table=safepointIndices();memcpy(table,si,safepointIndexEntries_*sizeof(SafepointIndex));}voidIonScript::copyOsiIndices(constOsiIndex*oi,MacroAssembler&masm){memcpy(osiIndices(),oi,osiIndexEntries_*sizeof(OsiIndex));}voidIonScript::copyRuntimeData(constuint8_t*data){memcpy(runtimeData(),data,runtimeSize());}voidIonScript::copyICEntries(constuint32_t*icEntries,MacroAssembler&masm){memcpy(icIndex(),icEntries,numICs()*sizeof(uint32_t));// Jumps in the caches reflect the offset of those jumps in the compiled// code, not the absolute positions of the jumps. Update according to the// final code address now.for(size_ti=0;i<numICs();i++)getICFromIndex(i).updateBaseAddress(method_,masm);}constSafepointIndex*IonScript::getSafepointIndex(uint32_tdisp)const{MOZ_ASSERT(safepointIndexEntries_>0);constSafepointIndex*table=safepointIndices();if(safepointIndexEntries_==1){MOZ_ASSERT(disp==table[0].displacement());return&table[0];}size_tminEntry=0;size_tmaxEntry=safepointIndexEntries_-1;uint32_tmin=table[minEntry].displacement();uint32_tmax=table[maxEntry].displacement();// Raise if the element is not in the list.MOZ_ASSERT(min<=disp&&disp<=max);// Approximate the location of the FrameInfo.size_tguess=(disp-min)*(maxEntry-minEntry)/(max-min)+minEntry;uint32_tguessDisp=table[guess].displacement();if(table[guess].displacement()==disp)return&table[guess];// Doing a linear scan from the guess should be more efficient in case of// small group which are equally distributed on the code.//// such as: <... ... ... ... . ... ...>if(guessDisp>disp){while(--guess>=minEntry){guessDisp=table[guess].displacement();MOZ_ASSERT(guessDisp>=disp);if(guessDisp==disp)return&table[guess];}}else{while(++guess<=maxEntry){guessDisp=table[guess].displacement();MOZ_ASSERT(guessDisp<=disp);if(guessDisp==disp)return&table[guess];}}MOZ_CRASH("displacement not found.");}constOsiIndex*IonScript::getOsiIndex(uint32_tdisp)const{constOsiIndex*end=osiIndices()+osiIndexEntries_;for(constOsiIndex*it=osiIndices();it!=end;++it){if(it->returnPointDisplacement()==disp)returnit;}MOZ_CRASH("Failed to find OSI point return address");}constOsiIndex*IonScript::getOsiIndex(uint8_t*retAddr)const{JitSpew(JitSpew_IonInvalidate,"IonScript %p has method %p raw %p",(void*)this,(void*)method(),method()->raw());MOZ_ASSERT(containsCodeAddress(retAddr));uint32_tdisp=retAddr-method()->raw();returngetOsiIndex(disp);}voidIonScript::Trace(JSTracer*trc,IonScript*script){if(script!=ION_DISABLED_SCRIPT)script->trace(trc);}voidIonScript::Destroy(FreeOp*fop,IonScript*script){script->unlinkFromRuntime(fop);/* * When the script contains pointers to nursery things, the store buffer can * contain entries that point into the fallback stub space. Since we can * destroy scripts outside the context of a GC, this situation could result * in us trying to mark invalid store buffer entries. * * Defer freeing any allocated blocks until after the next minor GC. */script->fallbackStubSpace_.freeAllAfterMinorGC(script->method()->zone());fop->delete_(script);}voidJS::DeletePolicy<js::jit::IonScript>::operator()(constjs::jit::IonScript*script){IonScript::Destroy(rt_->defaultFreeOp(),const_cast<IonScript*>(script));}voidIonScript::purgeOptimizedStubs(Zone*zone){for(size_ti=0;i<numSharedStubs();i++){IonICEntry&entry=sharedStubList()[i];if(!entry.hasStub())continue;ICStub*lastStub=entry.firstStub();while(lastStub->next())lastStub=lastStub->next();if(lastStub->isFallback()){// Unlink all stubs allocated in the optimized space.ICStub*stub=entry.firstStub();ICStub*prev=nullptr;while(stub->next()){if(!stub->allocatedInFallbackSpace()){lastStub->toFallbackStub()->unlinkStub(zone,prev,stub);stub=stub->next();continue;}prev=stub;stub=stub->next();}lastStub->toFallbackStub()->setInvalid();if(lastStub->isMonitoredFallback()){// Monitor stubs can't make calls, so are always in the// optimized stub space.ICTypeMonitor_Fallback*lastMonStub=lastStub->toMonitoredFallbackStub()->fallbackMonitorStub();lastMonStub->resetMonitorStubChain(zone);lastMonStub->setInvalid();}}elseif(lastStub->isTypeMonitor_Fallback()){lastStub->toTypeMonitor_Fallback()->resetMonitorStubChain(zone);lastStub->toTypeMonitor_Fallback()->setInvalid();}else{MOZ_ASSERT(lastStub->isTableSwitch());}}#ifdef DEBUG// All remaining stubs must be allocated in the fallback space.for(size_ti=0;i<numSharedStubs();i++){IonICEntry&entry=sharedStubList()[i];if(!entry.hasStub())continue;ICStub*stub=entry.firstStub();while(stub->next()){MOZ_ASSERT(stub->allocatedInFallbackSpace());stub=stub->next();}}#endif}voidIonScript::purgeICs(Zone*zone){for(size_ti=0;i<numICs();i++)getICFromIndex(i).reset(zone);}voidIonScript::unlinkFromRuntime(FreeOp*fop){// The writes to the executable buffer below may clobber backedge jumps, so// make sure that those backedges are unlinked from the runtime and not// reclobbered with garbage if an interrupt is requested.JitZoneGroup*jzg=method()->zone()->group()->jitZoneGroup;JitRuntime::AutoPreventBackedgePatchingapbp(fop->runtime());for(size_ti=0;i<backedgeEntries_;i++)jzg->removePatchableBackedge(fop->runtime()->jitRuntime(),&backedgeList()[i]);// Clear the list of backedges, so that this method is idempotent. It is// called during destruction, and may be additionally called when the// script is invalidated.backedgeEntries_=0;}namespacejs{namespacejit{staticvoidOptimizeSinCos(MIRGenerator*mir,MIRGraph&graph){// Now, we are looking for:// var y = sin(x);// var z = cos(x);// Graph before:// - 1 op// - 6 mathfunction op1 Sin// - 7 mathfunction op1 Cos// Graph will look like:// - 1 op// - 5 sincos op1// - 6 mathfunction sincos5 Sin// - 7 mathfunction sincos5 Cosfor(MBasicBlockIteratorblock(graph.begin());block!=graph.end();block++){for(MInstructionIteratoriter(block->begin()),end(block->end());iter!=end;){MInstruction*ins=*iter++;if(!ins->isMathFunction()||ins->isRecoveredOnBailout())continue;MMathFunction*insFunc=ins->toMathFunction();if(insFunc->function()!=MMathFunction::Sin&&insFunc->function()!=MMathFunction::Cos)continue;// Check if sin/cos is already optimized.if(insFunc->getOperand(0)->type()==MIRType::SinCosDouble)continue;// insFunc is either a |sin(x)| or |cos(x)| instruction. The// following loop iterates over the uses of |x| to check if both// |sin(x)| and |cos(x)| instructions exist.boolhasSin=false;boolhasCos=false;for(MUseDefIteratoruses(insFunc->input());uses;uses++){if(!uses.def()->isInstruction())continue;// We should replacing the argument of the sin/cos just when it// is dominated by the |block|.if(!block->dominates(uses.def()->block()))continue;MInstruction*insUse=uses.def()->toInstruction();if(!insUse->isMathFunction()||insUse->isRecoveredOnBailout())continue;MMathFunction*mathIns=insUse->toMathFunction();if(!hasSin&&mathIns->function()==MMathFunction::Sin){hasSin=true;JitSpew(JitSpew_Sincos,"Found sin in block %d.",mathIns->block()->id());}elseif(!hasCos&&mathIns->function()==MMathFunction::Cos){hasCos=true;JitSpew(JitSpew_Sincos,"Found cos in block %d.",mathIns->block()->id());}if(hasCos&&hasSin)break;}if(!hasCos||!hasSin){JitSpew(JitSpew_Sincos,"No sin/cos pair found.");continue;}JitSpew(JitSpew_Sincos,"Found, at least, a pair sin/cos. Adding sincos in block %d",block->id());// Adding the MSinCos and replacing the parameters of the// sin(x)/cos(x) to sin(sincos(x))/cos(sincos(x)).MSinCos*insSinCos=MSinCos::New(graph.alloc(),insFunc->input(),insFunc->toMathFunction()->cache());insSinCos->setImplicitlyUsedUnchecked();block->insertBefore(insFunc,insSinCos);for(MUseDefIteratoruses(insFunc->input());uses;){MDefinition*def=uses.def();uses++;if(!def->isInstruction())continue;// We should replacing the argument of the sin/cos just when it// is dominated by the |block|.if(!block->dominates(def->block()))continue;MInstruction*insUse=def->toInstruction();if(!insUse->isMathFunction()||insUse->isRecoveredOnBailout())continue;MMathFunction*mathIns=insUse->toMathFunction();if(mathIns->function()!=MMathFunction::Sin&&mathIns->function()!=MMathFunction::Cos)continue;mathIns->replaceOperand(0,insSinCos);JitSpew(JitSpew_Sincos,"Replacing %s by sincos in block %d",mathIns->function()==MMathFunction::Sin?"sin":"cos",mathIns->block()->id());}}}}boolOptimizeMIR(MIRGenerator*mir){MIRGraph&graph=mir->graph();GraphSpewer&gs=mir->graphSpewer();TraceLoggerThread*logger=TraceLoggerForCurrentThread();if(mir->shouldCancel("Start"))returnfalse;if(!mir->compilingWasm()){if(!MakeMRegExpHoistable(mir,graph))returnfalse;if(mir->shouldCancel("Make MRegExp Hoistable"))returnfalse;}gs.spewPass("BuildSSA");AssertBasicGraphCoherency(graph);if(!JitOptions.disablePgo&&!mir->compilingWasm()){AutoTraceLoglog(logger,TraceLogger_PruneUnusedBranches);if(!PruneUnusedBranches(mir,graph))returnfalse;gs.spewPass("Prune Unused Branches");AssertBasicGraphCoherency(graph);if(mir->shouldCancel("Prune Unused Branches"))returnfalse;}{AutoTraceLoglog(logger,TraceLogger_FoldEmptyBlocks);if(!FoldEmptyBlocks(graph))returnfalse;gs.spewPass("Fold Empty Blocks");AssertBasicGraphCoherency(graph);if(mir->shouldCancel("Fold Empty Blocks"))returnfalse;}{AutoTraceLoglog(logger,TraceLogger_FoldTests);if(!FoldTests(graph))returnfalse;gs.spewPass("Fold Tests");AssertBasicGraphCoherency(graph);if(mir->shouldCancel("Fold Tests"))returnfalse;}{AutoTraceLoglog(logger,TraceLogger_SplitCriticalEdges);if(!SplitCriticalEdges(graph))returnfalse;gs.spewPass("Split Critical Edges");AssertGraphCoherency(graph);if(mir->shouldCancel("Split Critical Edges"))returnfalse;}{AutoTraceLoglog(logger,TraceLogger_RenumberBlocks);RenumberBlocks(graph);gs.spewPass("Renumber Blocks");AssertGraphCoherency(graph);if(mir->shouldCancel("Renumber Blocks"))returnfalse;}{AutoTraceLoglog(logger,TraceLogger_DominatorTree);if(!BuildDominatorTree(graph))returnfalse;// No spew: graph not changed.if(mir->shouldCancel("Dominator Tree"))returnfalse;}{AutoTraceLoglog(logger,TraceLogger_PhiAnalysis);// Aggressive phi elimination must occur before any code elimination. If the// script contains a try-statement, we only compiled the try block and not// the catch or finally blocks, so in this case it's also invalid to use// aggressive phi elimination.Observabilityobservability=graph.hasTryBlock()?ConservativeObservability:AggressiveObservability;if(!EliminatePhis(mir,graph,observability))returnfalse;gs.spewPass("Eliminate phis");AssertGraphCoherency(graph);if(mir->shouldCancel("Eliminate phis"))returnfalse;if(!BuildPhiReverseMapping(graph))returnfalse;AssertExtendedGraphCoherency(graph);// No spew: graph not changed.if(mir->shouldCancel("Phi reverse mapping"))returnfalse;}if(!JitOptions.disableRecoverIns&&mir->optimizationInfo().scalarReplacementEnabled()){AutoTraceLoglog(logger,TraceLogger_ScalarReplacement);if(!ScalarReplacement(mir,graph))returnfalse;gs.spewPass("Scalar Replacement");AssertGraphCoherency(graph);if(mir->shouldCancel("Scalar Replacement"))returnfalse;}if(!mir->compilingWasm()){AutoTraceLoglog(logger,TraceLogger_ApplyTypes);if(!ApplyTypeInformation(mir,graph))returnfalse;gs.spewPass("Apply types");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("Apply types"))returnfalse;}if(!JitOptions.disableRecoverIns&&mir->optimizationInfo().eagerSimdUnboxEnabled()){AutoTraceLoglog(logger,TraceLogger_EagerSimdUnbox);if(!EagerSimdUnbox(mir,graph))returnfalse;gs.spewPass("Eager Simd Unbox");AssertGraphCoherency(graph);if(mir->shouldCancel("Eager Simd Unbox"))returnfalse;}if(mir->optimizationInfo().amaEnabled()){AutoTraceLoglog(logger,TraceLogger_AlignmentMaskAnalysis);AlignmentMaskAnalysisama(graph);if(!ama.analyze())returnfalse;gs.spewPass("Alignment Mask Analysis");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("Alignment Mask Analysis"))returnfalse;}ValueNumberergvn(mir,graph);if(!gvn.init())returnfalse;// Alias analysis is required for LICM and GVN so that we don't move// loads across stores.if(mir->optimizationInfo().licmEnabled()||mir->optimizationInfo().gvnEnabled()){{AutoTraceLoglog(logger,TraceLogger_AliasAnalysis);if(JitOptions.disableFlowAA){AliasAnalysisanalysis(mir,graph);if(!analysis.analyze())returnfalse;}else{FlowAliasAnalysisanalysis(mir,graph);if(!analysis.analyze())returnfalse;}gs.spewPass("Alias analysis");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("Alias analysis"))returnfalse;}if(!mir->compilingWasm()){// Eliminating dead resume point operands requires basic block// instructions to be numbered. Reuse the numbering computed during// alias analysis.if(!EliminateDeadResumePointOperands(mir,graph))returnfalse;if(mir->shouldCancel("Eliminate dead resume point operands"))returnfalse;}}if(mir->optimizationInfo().gvnEnabled()){AutoTraceLoglog(logger,TraceLogger_GVN);if(!gvn.run(ValueNumberer::UpdateAliasAnalysis))returnfalse;gs.spewPass("GVN");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("GVN"))returnfalse;}if(mir->optimizationInfo().licmEnabled()){AutoTraceLoglog(logger,TraceLogger_LICM);// LICM can hoist instructions from conditional branches and trigger// repeated bailouts. Disable it if this script is known to bailout// frequently.JSScript*script=mir->info().script();if(!script||!script->hadFrequentBailouts()){if(!LICM(mir,graph))returnfalse;gs.spewPass("LICM");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("LICM"))returnfalse;}}RangeAnalysisr(mir,graph);if(mir->optimizationInfo().rangeAnalysisEnabled()){AutoTraceLoglog(logger,TraceLogger_RangeAnalysis);if(!r.addBetaNodes())returnfalse;gs.spewPass("Beta");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("RA Beta"))returnfalse;if(!r.analyze()||!r.addRangeAssertions())returnfalse;gs.spewPass("Range Analysis");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("Range Analysis"))returnfalse;if(!r.removeBetaNodes())returnfalse;gs.spewPass("De-Beta");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("RA De-Beta"))returnfalse;if(mir->optimizationInfo().gvnEnabled()){boolshouldRunUCE=false;if(!r.prepareForUCE(&shouldRunUCE))returnfalse;gs.spewPass("RA check UCE");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("RA check UCE"))returnfalse;if(shouldRunUCE){if(!gvn.run(ValueNumberer::DontUpdateAliasAnalysis))returnfalse;gs.spewPass("UCE After RA");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("UCE After RA"))returnfalse;}}if(mir->optimizationInfo().autoTruncateEnabled()){if(!r.truncate())returnfalse;gs.spewPass("Truncate Doubles");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("Truncate Doubles"))returnfalse;}if(mir->optimizationInfo().loopUnrollingEnabled()){AutoTraceLoglog(logger,TraceLogger_LoopUnrolling);if(!UnrollLoops(graph,r.loopIterationBounds))returnfalse;gs.spewPass("Unroll Loops");AssertExtendedGraphCoherency(graph);}}if(!JitOptions.disableRecoverIns){AutoTraceLoglog(logger,TraceLogger_Sink);if(!Sink(mir,graph))returnfalse;gs.spewPass("Sink");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("Sink"))returnfalse;}if(!JitOptions.disableRecoverIns&&mir->optimizationInfo().rangeAnalysisEnabled()){AutoTraceLoglog(logger,TraceLogger_RemoveUnnecessaryBitops);if(!r.removeUnnecessaryBitops())returnfalse;gs.spewPass("Remove Unnecessary Bitops");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("Remove Unnecessary Bitops"))returnfalse;}{AutoTraceLoglog(logger,TraceLogger_FoldLinearArithConstants);if(!FoldLinearArithConstants(mir,graph))returnfalse;gs.spewPass("Fold Linear Arithmetic Constants");AssertBasicGraphCoherency(graph);if(mir->shouldCancel("Fold Linear Arithmetic Constants"))returnfalse;}if(mir->optimizationInfo().eaaEnabled()){AutoTraceLoglog(logger,TraceLogger_EffectiveAddressAnalysis);EffectiveAddressAnalysiseaa(mir,graph);if(!eaa.analyze())returnfalse;gs.spewPass("Effective Address Analysis");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("Effective Address Analysis"))returnfalse;}if(mir->optimizationInfo().sincosEnabled()){AutoTraceLoglog(logger,TraceLogger_Sincos);OptimizeSinCos(mir,graph);gs.spewPass("Sincos optimization");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("Sincos optimization"))returnfalse;}// BCE marks bounds checks as dead, so do BCE before DCE.if(mir->compilingWasm()&&!JitOptions.wasmAlwaysCheckBounds){if(!EliminateBoundsChecks(mir,graph))returnfalse;gs.spewPass("Redundant Bounds Check Elimination");AssertGraphCoherency(graph);if(mir->shouldCancel("BCE"))returnfalse;}{AutoTraceLoglog(logger,TraceLogger_EliminateDeadCode);if(!EliminateDeadCode(mir,graph))returnfalse;gs.spewPass("DCE");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("DCE"))returnfalse;}if(mir->optimizationInfo().instructionReorderingEnabled()){AutoTraceLoglog(logger,TraceLogger_ReorderInstructions);if(!ReorderInstructions(mir,graph))returnfalse;gs.spewPass("Reordering");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("Reordering"))returnfalse;}// Make loops contiguous. We do this after GVN/UCE and range analysis,// which can remove CFG edges, exposing more blocks that can be moved.{AutoTraceLoglog(logger,TraceLogger_MakeLoopsContiguous);if(!MakeLoopsContiguous(graph))returnfalse;gs.spewPass("Make loops contiguous");AssertExtendedGraphCoherency(graph);if(mir->shouldCancel("Make loops contiguous"))returnfalse;}AssertExtendedGraphCoherency(graph,/* underValueNumberer = */false,/* force = */true);// Passes after this point must not move instructions; these analyses// depend on knowing the final order in which instructions will execute.if(mir->optimizationInfo().edgeCaseAnalysisEnabled()){AutoTraceLoglog(logger,TraceLogger_EdgeCaseAnalysis);EdgeCaseAnalysisedgeCaseAnalysis(mir,graph);if(!edgeCaseAnalysis.analyzeLate())returnfalse;gs.spewPass("Edge Case Analysis (Late)");AssertGraphCoherency(graph);if(mir->shouldCancel("Edge Case Analysis (Late)"))returnfalse;}if(mir->optimizationInfo().eliminateRedundantChecksEnabled()){AutoTraceLoglog(logger,TraceLogger_EliminateRedundantChecks);// Note: check elimination has to run after all other passes that move// instructions. Since check uses are replaced with the actual index,// code motion after this pass could incorrectly move a load or store// before its bounds check.if(!EliminateRedundantChecks(graph))returnfalse;gs.spewPass("Bounds Check Elimination");AssertGraphCoherency(graph);}if(!mir->compilingWasm()){AutoTraceLoglog(logger,TraceLogger_AddKeepAliveInstructions);if(!AddKeepAliveInstructions(graph))returnfalse;gs.spewPass("Add KeepAlive Instructions");AssertGraphCoherency(graph);}AssertGraphCoherency(graph,/* force = */true);DumpMIRExpressions(graph);returntrue;}LIRGraph*GenerateLIR(MIRGenerator*mir){MIRGraph&graph=mir->graph();GraphSpewer&gs=mir->graphSpewer();TraceLoggerThread*logger=TraceLoggerForCurrentThread();LIRGraph*lir=mir->alloc().lifoAlloc()->new_<LIRGraph>(&graph);if(!lir||!lir->init())returnnullptr;LIRGeneratorlirgen(mir,graph,*lir);{AutoTraceLoglog(logger,TraceLogger_GenerateLIR);if(!lirgen.generate())returnnullptr;gs.spewPass("Generate LIR");if(mir->shouldCancel("Generate LIR"))returnnullptr;}AllocationIntegrityStateintegrity(*lir);{AutoTraceLoglog(logger,TraceLogger_RegisterAllocation);IonRegisterAllocatorallocator=mir->optimizationInfo().registerAllocator();switch(allocator){caseRegisterAllocator_Backtracking:caseRegisterAllocator_Testbed:{#ifdef DEBUGif(JitOptions.fullDebugChecks){if(!integrity.record())returnnullptr;}#endifBacktrackingAllocatorregalloc(mir,&lirgen,*lir,allocator==RegisterAllocator_Testbed);if(!regalloc.go())returnnullptr;#ifdef DEBUGif(JitOptions.fullDebugChecks){if(!integrity.check(false))returnnullptr;}#endifgs.spewPass("Allocate Registers [Backtracking]");break;}caseRegisterAllocator_Stupid:{// Use the integrity checker to populate safepoint information, so// run it in all builds.if(!integrity.record())returnnullptr;StupidAllocatorregalloc(mir,&lirgen,*lir);if(!regalloc.go())returnnullptr;if(!integrity.check(true))returnnullptr;gs.spewPass("Allocate Registers [Stupid]");break;}default:MOZ_CRASH("Bad regalloc");}if(mir->shouldCancel("Allocate Registers"))returnnullptr;}returnlir;}CodeGenerator*GenerateCode(MIRGenerator*mir,LIRGraph*lir){TraceLoggerThread*logger=TraceLoggerForCurrentThread();AutoTraceLoglog(logger,TraceLogger_GenerateCode);CodeGenerator*codegen=js_new<CodeGenerator>(mir,lir);if(!codegen)returnnullptr;if(!codegen->generate()){js_delete(codegen);returnnullptr;}returncodegen;}CodeGenerator*CompileBackEnd(MIRGenerator*mir){// Everything in CompileBackEnd can potentially run on a helper thread.AutoEnterIonCompilationenter(mir->safeForMinorGC());AutoSpewEndFunctionspewEndFunction(mir);if(!OptimizeMIR(mir))returnnullptr;LIRGraph*lir=GenerateLIR(mir);if(!lir)returnnullptr;returnGenerateCode(mir,lir);}// Find a builder which the current thread can finish.staticIonBuilder*GetFinishedBuilder(ZoneGroup*group,GlobalHelperThreadState::IonBuilderVector&finished){for(size_ti=0;i<finished.length();i++){IonBuilder*testBuilder=finished[i];if(testBuilder->script()->runtimeFromAnyThread()==group->runtime&&testBuilder->script()->zone()->group()==group){HelperThreadState().remove(finished,&i);group->numFinishedBuilders--;returntestBuilder;}}returnnullptr;}voidAttachFinishedCompilations(ZoneGroup*group,JSContext*maybecx){MOZ_ASSERT_IF(maybecx,maybecx->zone()->group()==group);if(!group->numFinishedBuilders)return;AutoLockHelperThreadStatelock;GlobalHelperThreadState::IonBuilderVector&finished=HelperThreadState().ionFinishedList(lock);// Incorporate any off thread compilations for the runtime which have// finished, failed or have been cancelled.while(true){// Find a finished builder for the zone group.IonBuilder*builder=GetFinishedBuilder(group,finished);if(!builder)break;JSScript*script=builder->script();MOZ_ASSERT(script->hasBaselineScript());script->baselineScript()->setPendingIonBuilder(group->runtime,script,builder);group->ionLazyLinkListAdd(builder);// Don't keep more than 100 lazy link builders in a zone group.// Link the oldest ones immediately. Only do this if we have a valid// context to use (otherwise this method might have been called in the// middle of a compartment change on the current thread's context).if(maybecx){while(group->ionLazyLinkListSize()>100){jit::IonBuilder*builder=group->ionLazyLinkList().getLast();RootedScriptscript(maybecx,builder->script());AutoUnlockHelperThreadStateunlock(lock);AutoCompartmentac(maybecx,script);jit::LinkIonScript(maybecx,script);}}}MOZ_ASSERT(!group->numFinishedBuilders);}staticvoidTrackAllProperties(JSContext*cx,JSObject*obj){MOZ_ASSERT(obj->isSingleton());for(Shape::Range<NoGC>range(obj->as<NativeObject>().lastProperty());!range.empty();range.popFront())EnsureTrackPropertyTypes(cx,obj,range.front().propid());}staticvoidTrackPropertiesForSingletonScopes(JSContext*cx,JSScript*script,BaselineFrame*baselineFrame){// Ensure that all properties of singleton call objects which the script// could access are tracked. These are generally accessed through// ALIASEDVAR operations in baseline and will not be tracked even if they// have been accessed in baseline code.JSObject*environment=script->functionNonDelazifying()?script->functionNonDelazifying()->environment():nullptr;while(environment&&!environment->is<GlobalObject>()){if(environment->is<CallObject>()&&environment->isSingleton())TrackAllProperties(cx,environment);environment=environment->enclosingEnvironment();}if(baselineFrame){JSObject*scope=baselineFrame->environmentChain();if(scope->is<CallObject>()&&scope->isSingleton())TrackAllProperties(cx,scope);}}staticvoidTrackIonAbort(JSContext*cx,JSScript*script,jsbytecode*pc,constchar*message){if(!cx->runtime()->jitRuntime()->isOptimizationTrackingEnabled(cx->zone()->group()))return;// Only bother tracking aborts of functions we're attempting to// Ion-compile after successfully running in Baseline.if(!script->hasBaselineScript())return;JitcodeGlobalTable*table=cx->runtime()->jitRuntime()->getJitcodeGlobalTable();void*ptr=script->baselineScript()->method()->raw();JitcodeGlobalEntry&entry=table->lookupInfallible(ptr);entry.baselineEntry().trackIonAbort(pc,message);}staticvoidTrackAndSpewIonAbort(JSContext*cx,JSScript*script,constchar*message){JitSpew(JitSpew_IonAbort,"%s",message);TrackIonAbort(cx,script,script->code(),message);}staticAbortReasonIonCompile(JSContext*cx,JSScript*script,BaselineFrame*baselineFrame,jsbytecode*osrPc,boolrecompile,OptimizationLeveloptimizationLevel){TraceLoggerThread*logger=TraceLoggerForCurrentThread(cx);TraceLoggerEventevent(TraceLogger_AnnotateScripts,script);AutoTraceLoglogScript(logger,event);AutoTraceLoglogCompile(logger,TraceLogger_IonCompilation);// Make sure the script's canonical function isn't lazy. We can't de-lazify// it in a helper thread.script->ensureNonLazyCanonicalFunction();TrackPropertiesForSingletonScopes(cx,script,baselineFrame);LifoAlloc*alloc=cx->new_<LifoAlloc>(TempAllocator::PreferredLifoChunkSize);if(!alloc)returnAbortReason::Alloc;ScopedJSDeletePtr<LifoAlloc>autoDelete(alloc);TempAllocator*temp=alloc->new_<TempAllocator>(alloc);if(!temp)returnAbortReason::Alloc;JitContextjctx(cx,temp);if(!cx->compartment()->ensureJitCompartmentExists(cx))returnAbortReason::Alloc;if(!cx->compartment()->jitCompartment()->ensureIonStubsExist(cx))returnAbortReason::Alloc;MIRGraph*graph=alloc->new_<MIRGraph>(temp);if(!graph)returnAbortReason::Alloc;InlineScriptTree*inlineScriptTree=InlineScriptTree::New(temp,nullptr,nullptr,script);if(!inlineScriptTree)returnAbortReason::Alloc;CompileInfo*info=alloc->new_<CompileInfo>(script,script->functionNonDelazifying(),osrPc,Analysis_None,script->needsArgsObj(),inlineScriptTree);if(!info)returnAbortReason::Alloc;BaselineInspector*inspector=alloc->new_<BaselineInspector>(script);if(!inspector)returnAbortReason::Alloc;BaselineFrameInspector*baselineFrameInspector=nullptr;if(baselineFrame){baselineFrameInspector=NewBaselineFrameInspector(temp,baselineFrame,info);if(!baselineFrameInspector)returnAbortReason::Alloc;}CompilerConstraintList*constraints=NewCompilerConstraintList(*temp);if(!constraints)returnAbortReason::Alloc;constOptimizationInfo*optimizationInfo=IonOptimizations.get(optimizationLevel);constJitCompileOptionsoptions(cx);IonBuilder*builder=alloc->new_<IonBuilder>((JSContext*)nullptr,CompileCompartment::get(cx->compartment()),options,temp,graph,constraints,inspector,info,optimizationInfo,baselineFrameInspector);if(!builder)returnAbortReason::Alloc;if(cx->zone()->group()->storeBuffer().cancelIonCompilations())builder->setNotSafeForMinorGC();MOZ_ASSERT(recompile==builder->script()->hasIonScript());MOZ_ASSERT(builder->script()->canIonCompile());RootedScriptbuilderScript(cx,builder->script());if(recompile)builderScript->ionScript()->setRecompiling();SpewBeginFunction(builder,builderScript);AbortReasonOr<Ok>buildResult=Ok();{AutoEnterAnalysisenter(cx);buildResult=builder->build();builder->clearForBackEnd();}if(buildResult.isErr()){AbortReasonreason=buildResult.unwrapErr();builder->graphSpewer().endFunction();if(reason==AbortReason::PreliminaryObjects){// Some group was accessed which has associated preliminary objects// to analyze. Do this now and we will try to build again shortly.constMIRGenerator::ObjectGroupVector&groups=builder->abortedPreliminaryGroups();for(size_ti=0;i<groups.length();i++){ObjectGroup*group=groups[i];if(group->newScript()){if(!group->newScript()->maybeAnalyze(cx,group,nullptr,/* force = */true))returnAbortReason::Alloc;}elseif(group->maybePreliminaryObjects()){group->maybePreliminaryObjects()->maybeAnalyze(cx,group,/* force = */true);}else{MOZ_CRASH("Unexpected aborted preliminary group");}}}if(builder->hadActionableAbort()){JSScript*abortScript;jsbytecode*abortPc;constchar*abortMessage;builder->actionableAbortLocationAndMessage(&abortScript,&abortPc,&abortMessage);TrackIonAbort(cx,abortScript,abortPc,abortMessage);}if(cx->isThrowingOverRecursed()){// Non-analysis compilations should never fail with stack overflow.MOZ_CRASH("Stack overflow during compilation");}returnreason;}AssertBasicGraphCoherency(builder->graph());// If possible, compile the script off thread.if(options.offThreadCompilationAvailable()){JitSpew(JitSpew_IonSyncLogs,"Can't log script %s:%"PRIuSIZE". (Compiled on background thread.)",builderScript->filename(),builderScript->lineno());if(!CreateMIRRootList(*builder))returnAbortReason::Alloc;if(!StartOffThreadIonCompile(cx,builder)){JitSpew(JitSpew_IonAbort,"Unable to start off-thread ion compilation.");builder->graphSpewer().endFunction();returnAbortReason::Alloc;}if(!recompile)builderScript->setIonScript(cx->runtime(),ION_COMPILING_SCRIPT);// The allocator and associated data will be destroyed after being// processed in the finishedOffThreadCompilations list.autoDelete.forget();returnAbortReason::NoAbort;}boolsucceeded=false;{ScopedJSDeletePtr<CodeGenerator>codegen;AutoEnterAnalysisenter(cx);codegen=CompileBackEnd(builder);if(!codegen){JitSpew(JitSpew_IonAbort,"Failed during back-end compilation.");if(cx->isExceptionPending())returnAbortReason::Error;returnAbortReason::Disable;}succeeded=LinkCodeGen(cx,builder,codegen);}if(succeeded)returnAbortReason::NoAbort;if(cx->isExceptionPending())returnAbortReason::Error;returnAbortReason::Disable;}staticboolCheckFrame(JSContext*cx,BaselineFrame*frame){MOZ_ASSERT(!frame->script()->isStarGenerator());MOZ_ASSERT(!frame->script()->isLegacyGenerator());MOZ_ASSERT(!frame->script()->isAsync());MOZ_ASSERT(!frame->isDebuggerEvalFrame());MOZ_ASSERT(!frame->isEvalFrame());// This check is to not overrun the stack.if(frame->isFunctionFrame()){if(TooManyActualArguments(frame->numActualArgs())){TrackAndSpewIonAbort(cx,frame->script(),"too many actual arguments");returnfalse;}if(TooManyFormalArguments(frame->numFormalArgs())){TrackAndSpewIonAbort(cx,frame->script(),"too many arguments");returnfalse;}}returntrue;}staticboolCheckScript(JSContext*cx,JSScript*script,boolosr){if(script->isForEval()){// Eval frames are not yet supported. Supporting this will require new// logic in pushBailoutFrame to deal with linking prev.// Additionally, JSOP_DEFVAR support will require baking in isEvalFrame().TrackAndSpewIonAbort(cx,script,"eval script");returnfalse;}if(script->isStarGenerator()||script->isLegacyGenerator()){TrackAndSpewIonAbort(cx,script,"generator script");returnfalse;}if(script->isAsync()){TrackAndSpewIonAbort(cx,script,"async script");returnfalse;}if(script->hasNonSyntacticScope()&&!script->functionNonDelazifying()){// Support functions with a non-syntactic global scope but not other// scripts. For global scripts, IonBuilder currently uses the global// object as scope chain, this is not valid when the script has a// non-syntactic global scope.TrackAndSpewIonAbort(cx,script,"has non-syntactic global scope");returnfalse;}if(script->functionHasExtraBodyVarScope()&&script->functionExtraBodyVarScope()->hasEnvironment()){// This restriction will be lifted when intra-function scope chains// are compilable by Ion. See bug 1273858.TrackAndSpewIonAbort(cx,script,"has extra var environment");returnfalse;}if(script->nTypeSets()>=UINT16_MAX){// In this case multiple bytecode ops can share a single observed// TypeSet (see bug 1303710).TrackAndSpewIonAbort(cx,script,"too many typesets");returnfalse;}returntrue;}staticMethodStatusCheckScriptSize(JSContext*cx,JSScript*script){if(!JitOptions.limitScriptSize)returnMethod_Compiled;uint32_tnumLocalsAndArgs=NumLocalsAndArgs(script);if(script->length()>MAX_ACTIVE_THREAD_SCRIPT_SIZE||numLocalsAndArgs>MAX_ACTIVE_THREAD_LOCALS_AND_ARGS){if(!OffThreadCompilationAvailable(cx)){JitSpew(JitSpew_IonAbort,"Script too large (%"PRIuSIZE" bytes) (%u locals/args)",script->length(),numLocalsAndArgs);TrackIonAbort(cx,script,script->code(),"too large");returnMethod_CantCompile;}}returnMethod_Compiled;}boolCanIonCompileScript(JSContext*cx,JSScript*script,boolosr){if(!script->canIonCompile()||!CheckScript(cx,script,osr))returnfalse;returnCheckScriptSize(cx,script)==Method_Compiled;}staticOptimizationLevelGetOptimizationLevel(HandleScriptscript,jsbytecode*pc){returnIonOptimizations.levelForScript(script,pc);}staticMethodStatusCompile(JSContext*cx,HandleScriptscript,BaselineFrame*osrFrame,jsbytecode*osrPc,boolforceRecompile=false){MOZ_ASSERT(jit::IsIonEnabled(cx));MOZ_ASSERT(jit::IsBaselineEnabled(cx));MOZ_ASSERT_IF(osrPc!=nullptr,LoopEntryCanIonOsr(osrPc));if(!script->hasBaselineScript())returnMethod_Skipped;if(script->isDebuggee()||(osrFrame&&osrFrame->isDebuggee())){TrackAndSpewIonAbort(cx,script,"debugging");returnMethod_Skipped;}if(!CheckScript(cx,script,bool(osrPc))){JitSpew(JitSpew_IonAbort,"Aborted compilation of %s:%"PRIuSIZE,script->filename(),script->lineno());returnMethod_CantCompile;}MethodStatusstatus=CheckScriptSize(cx,script);if(status!=Method_Compiled){JitSpew(JitSpew_IonAbort,"Aborted compilation of %s:%"PRIuSIZE,script->filename(),script->lineno());returnstatus;}boolrecompile=false;OptimizationLeveloptimizationLevel=GetOptimizationLevel(script,osrPc);if(optimizationLevel==OptimizationLevel::DontCompile)returnMethod_Skipped;if(!CanLikelyAllocateMoreExecutableMemory()){script->resetWarmUpCounter();returnMethod_Skipped;}if(script->hasIonScript()){IonScript*scriptIon=script->ionScript();if(!scriptIon->method())returnMethod_CantCompile;// Don't recompile/overwrite higher optimized code,// with a lower optimization level.if(optimizationLevel<=scriptIon->optimizationLevel()&&!forceRecompile)returnMethod_Compiled;// Don't start compiling if already compilingif(scriptIon->isRecompiling())returnMethod_Compiled;if(osrPc)scriptIon->resetOsrPcMismatchCounter();recompile=true;}if(script->baselineScript()->hasPendingIonBuilder()){IonBuilder*buildIon=script->baselineScript()->pendingIonBuilder();if(optimizationLevel<=buildIon->optimizationInfo().level()&&!forceRecompile)returnMethod_Compiled;recompile=true;}AbortReasonreason=IonCompile(cx,script,osrFrame,osrPc,recompile,optimizationLevel);if(reason==AbortReason::Error)returnMethod_Error;if(reason==AbortReason::Disable)returnMethod_CantCompile;if(reason==AbortReason::Alloc){ReportOutOfMemory(cx);returnMethod_Error;}// Compilation succeeded or we invalidated right away or an inlining/alloc abortif(script->hasIonScript())returnMethod_Compiled;returnMethod_Skipped;}}// namespace jit}// namespace jsbooljit::OffThreadCompilationAvailable(JSContext*cx){// Even if off thread compilation is enabled, compilation must still occur// on the active thread in some cases.//// Require cpuCount > 1 so that Ion compilation jobs and active-thread// execution are not competing for the same resources.returncx->runtime()->canUseOffthreadIonCompilation()&&HelperThreadState().cpuCount>1&&CanUseExtraThreads();}MethodStatusjit::CanEnter(JSContext*cx,RunState&state){MOZ_ASSERT(jit::IsIonEnabled(cx));JSScript*script=state.script();// Skip if the script has been disabled.if(!script->canIonCompile())returnMethod_Skipped;// Skip if the script is being compiled off thread.if(script->isIonCompilingOffThread())returnMethod_Skipped;// Skip if the code is expected to result in a bailout.if(script->hasIonScript()&&script->ionScript()->bailoutExpected())returnMethod_Skipped;RootedScriptrscript(cx,script);// If constructing, allocate a new |this| object before building Ion.// Creating |this| is done before building Ion because it may change the// type information and invalidate compilation results.if(state.isInvoke()){InvokeState&invoke=*state.asInvoke();if(TooManyActualArguments(invoke.args().length())){TrackAndSpewIonAbort(cx,script,"too many actual args");ForbidCompilation(cx,script);returnMethod_CantCompile;}if(TooManyFormalArguments(invoke.args().callee().as<JSFunction>().nargs())){TrackAndSpewIonAbort(cx,script,"too many args");ForbidCompilation(cx,script);returnMethod_CantCompile;}if(!state.maybeCreateThisForConstructor(cx)){if(cx->isThrowingOutOfMemory()){cx->recoverFromOutOfMemory();returnMethod_Skipped;}returnMethod_Error;}}// If --ion-eager is used, compile with Baseline first, so that we// can directly enter IonMonkey.if(JitOptions.eagerCompilation&&!rscript->hasBaselineScript()){MethodStatusstatus=CanEnterBaselineMethod(cx,state);if(status!=Method_Compiled)returnstatus;}// Skip if the script is being compiled off thread or can't be// Ion-compiled (again). MaybeCreateThisForConstructor could have// started an Ion compilation or marked the script as uncompilable.if(rscript->isIonCompilingOffThread()||!rscript->canIonCompile())returnMethod_Skipped;// Attempt compilation. Returns Method_Compiled if already compiled.MethodStatusstatus=Compile(cx,rscript,nullptr,nullptr);if(status!=Method_Compiled){if(status==Method_CantCompile)ForbidCompilation(cx,rscript);returnstatus;}if(state.script()->baselineScript()->hasPendingIonBuilder()){LinkIonScript(cx,state.script());if(!state.script()->hasIonScript())returnjit::Method_Skipped;}returnMethod_Compiled;}staticMethodStatusBaselineCanEnterAtEntry(JSContext*cx,HandleScriptscript,BaselineFrame*frame){MOZ_ASSERT(jit::IsIonEnabled(cx));MOZ_ASSERT(frame->callee()->nonLazyScript()->canIonCompile());MOZ_ASSERT(!frame->callee()->nonLazyScript()->isIonCompilingOffThread());MOZ_ASSERT(!frame->callee()->nonLazyScript()->hasIonScript());MOZ_ASSERT(frame->isFunctionFrame());// Mark as forbidden if frame can't be handled.if(!CheckFrame(cx,frame)){ForbidCompilation(cx,script);returnMethod_CantCompile;}// Attempt compilation. Returns Method_Compiled if already compiled.MethodStatusstatus=Compile(cx,script,frame,nullptr);if(status!=Method_Compiled){if(status==Method_CantCompile)ForbidCompilation(cx,script);returnstatus;}returnMethod_Compiled;}// Decide if a transition from baseline execution to Ion code should occur.// May compile or recompile the target JSScript.staticMethodStatusBaselineCanEnterAtBranch(JSContext*cx,HandleScriptscript,BaselineFrame*osrFrame,jsbytecode*pc){MOZ_ASSERT(jit::IsIonEnabled(cx));MOZ_ASSERT((JSOp)*pc==JSOP_LOOPENTRY);MOZ_ASSERT(LoopEntryCanIonOsr(pc));// Skip if the script has been disabled.if(!script->canIonCompile())returnMethod_Skipped;// Skip if the script is being compiled off thread.if(script->isIonCompilingOffThread())returnMethod_Skipped;// Skip if the code is expected to result in a bailout.if(script->hasIonScript()&&script->ionScript()->bailoutExpected())returnMethod_Skipped;// Optionally ignore on user request.if(!JitOptions.osr)returnMethod_Skipped;// Mark as forbidden if frame can't be handled.if(!CheckFrame(cx,osrFrame)){ForbidCompilation(cx,script);returnMethod_CantCompile;}// Check if the jitcode still needs to get linked and do this// to have a valid IonScript.if(script->baselineScript()->hasPendingIonBuilder())LinkIonScript(cx,script);// By default a recompilation doesn't happen on osr mismatch.// Decide if we want to force a recompilation if this happens too much.boolforce=false;if(script->hasIonScript()&&pc!=script->ionScript()->osrPc()){uint32_tcount=script->ionScript()->incrOsrPcMismatchCounter();if(count<=JitOptions.osrPcMismatchesBeforeRecompile)returnMethod_Skipped;force=true;}// Attempt compilation.// - Returns Method_Compiled if the right ionscript is present// (Meaning it was present or a sequantial compile finished)// - Returns Method_Skipped if pc doesn't match// (This means a background thread compilation with that pc could have started or not.)RootedScriptrscript(cx,script);MethodStatusstatus=Compile(cx,rscript,osrFrame,pc,force);if(status!=Method_Compiled){if(status==Method_CantCompile)ForbidCompilation(cx,script);returnstatus;}// Return the compilation was skipped when the osr pc wasn't adjusted.// This can happen when there was still an IonScript available and a// background compilation started, but hasn't finished yet.// Or when we didn't force a recompile.if(script->hasIonScript()&&pc!=script->ionScript()->osrPc())returnMethod_Skipped;returnMethod_Compiled;}booljit::IonCompileScriptForBaseline(JSContext*cx,BaselineFrame*frame,jsbytecode*pc){// A TI OOM will disable TI and Ion.if(!jit::IsIonEnabled(cx))returntrue;RootedScriptscript(cx,frame->script());boolisLoopEntry=JSOp(*pc)==JSOP_LOOPENTRY;MOZ_ASSERT(!isLoopEntry||LoopEntryCanIonOsr(pc));if(!script->canIonCompile()){// TODO: ASSERT that ion-compilation-disabled checker stub doesn't exist.// TODO: Clear all optimized stubs.// TODO: Add a ion-compilation-disabled checker IC stubscript->resetWarmUpCounter();returntrue;}MOZ_ASSERT(!script->isIonCompilingOffThread());// If Ion script exists, but PC is not at a loop entry, then Ion will be entered for// this script at an appropriate LOOPENTRY or the next time this function is called.if(script->hasIonScript()&&!isLoopEntry){JitSpew(JitSpew_BaselineOSR,"IonScript exists, but not at loop entry!");// TODO: ASSERT that a ion-script-already-exists checker stub doesn't exist.// TODO: Clear all optimized stubs.// TODO: Add a ion-script-already-exists checker stub.returntrue;}// Ensure that Ion-compiled code is available.JitSpew(JitSpew_BaselineOSR,"WarmUpCounter for %s:%"PRIuSIZE" reached %d at pc %p, trying to switch to Ion!",script->filename(),script->lineno(),(int)script->getWarmUpCount(),(void*)pc);MethodStatusstat;if(isLoopEntry){MOZ_ASSERT(LoopEntryCanIonOsr(pc));JitSpew(JitSpew_BaselineOSR," Compile at loop entry!");stat=BaselineCanEnterAtBranch(cx,script,frame,pc);}elseif(frame->isFunctionFrame()){JitSpew(JitSpew_BaselineOSR," Compile function from top for later entry!");stat=BaselineCanEnterAtEntry(cx,script,frame);}else{returntrue;}if(stat==Method_Error){JitSpew(JitSpew_BaselineOSR," Compile with Ion errored!");returnfalse;}if(stat==Method_CantCompile)JitSpew(JitSpew_BaselineOSR," Can't compile with Ion!");elseif(stat==Method_Skipped)JitSpew(JitSpew_BaselineOSR," Skipped compile with Ion!");elseif(stat==Method_Compiled)JitSpew(JitSpew_BaselineOSR," Compiled with Ion!");elseMOZ_CRASH("Invalid MethodStatus!");// Failed to compile. Reset warm-up counter and return.if(stat!=Method_Compiled){// TODO: If stat == Method_CantCompile, insert stub that just skips the// warm-up counter entirely, instead of resetting it.boolbailoutExpected=script->hasIonScript()&&script->ionScript()->bailoutExpected();if(stat==Method_CantCompile||bailoutExpected){JitSpew(JitSpew_BaselineOSR," Reset WarmUpCounter cantCompile=%s bailoutExpected=%s!",stat==Method_CantCompile?"yes":"no",bailoutExpected?"yes":"no");script->resetWarmUpCounter();}returntrue;}returntrue;}MethodStatusjit::Recompile(JSContext*cx,HandleScriptscript,BaselineFrame*osrFrame,jsbytecode*osrPc,boolforce){MOZ_ASSERT(script->hasIonScript());if(script->ionScript()->isRecompiling())returnMethod_Compiled;MethodStatusstatus=Compile(cx,script,osrFrame,osrPc,force);if(status!=Method_Compiled){if(status==Method_CantCompile)ForbidCompilation(cx,script);returnstatus;}returnMethod_Compiled;}MethodStatusjit::CanEnterUsingFastInvoke(JSContext*cx,HandleScriptscript,uint32_tnumActualArgs){MOZ_ASSERT(jit::IsIonEnabled(cx));// Skip if the code is expected to result in a bailout.if(!script->hasIonScript()||script->ionScript()->bailoutExpected())returnMethod_Skipped;// Don't handle arguments underflow, to make this work we would have to pad// missing arguments with |undefined|.if(numActualArgs<script->functionNonDelazifying()->nargs())returnMethod_Skipped;if(!cx->compartment()->ensureJitCompartmentExists(cx))returnMethod_Error;// This can GC, so afterward, script->ion is not guaranteed to be valid.if(!cx->runtime()->jitRuntime()->enterIon())returnMethod_Error;if(!script->hasIonScript())returnMethod_Skipped;returnMethod_Compiled;}staticJitExecStatusEnterIon(JSContext*cx,EnterJitData&data){if(!CheckRecursionLimit(cx))returnJitExec_Aborted;MOZ_ASSERT(jit::IsIonEnabled(cx));MOZ_ASSERT(!data.osrFrame);#ifdef DEBUG// See comment in EnterBaseline.mozilla::Maybe<JS::AutoAssertNoGC>nogc;nogc.emplace(cx);#endifEnterJitCodeenter=cx->runtime()->jitRuntime()->enterIon();// Caller must construct |this| before invoking the Ion function.MOZ_ASSERT_IF(data.constructing,data.maxArgv[0].isObject()||data.maxArgv[0].isMagic(JS_UNINITIALIZED_LEXICAL));data.result.setInt32(data.numActualArgs);{AssertCompartmentUnchangedpcc(cx);ActivationEntryMonitorentryMonitor(cx,data.calleeToken);JitActivationactivation(cx);#ifdef DEBUGnogc.reset();#endifCALL_GENERATED_CODE(enter,data.jitcode,data.maxArgc,data.maxArgv,/* osrFrame = */nullptr,data.calleeToken,/* envChain = */nullptr,0,data.result.address());}MOZ_ASSERT(!cx->hasIonReturnOverride());// Jit callers wrap primitive constructor return, except for derived class constructors.if(!data.result.isMagic()&&data.constructing&&data.result.isPrimitive()){MOZ_ASSERT(data.maxArgv[0].isObject());data.result=data.maxArgv[0];}// Release temporary buffer used for OSR into Ion.cx->freeOsrTempData();MOZ_ASSERT_IF(data.result.isMagic(),data.result.isMagic(JS_ION_ERROR));returndata.result.isMagic()?JitExec_Error:JitExec_Ok;}booljit::SetEnterJitData(JSContext*cx,EnterJitData&data,RunState&state,MutableHandle<GCVector<Value>>vals){data.osrFrame=nullptr;if(state.isInvoke()){constCallArgs&args=state.asInvoke()->args();unsignednumFormals=state.script()->functionNonDelazifying()->nargs();data.constructing=state.asInvoke()->constructing();data.numActualArgs=args.length();data.maxArgc=Max(args.length(),numFormals)+1;data.envChain=nullptr;data.calleeToken=CalleeToToken(&args.callee().as<JSFunction>(),data.constructing);if(data.numActualArgs>=numFormals){data.maxArgv=args.base()+1;}else{MOZ_ASSERT(vals.empty());unsignednumPushedArgs=Max(args.length(),numFormals);if(!vals.reserve(numPushedArgs+1+data.constructing))returnfalse;// Append |this| and any provided arguments.for(size_ti=1;i<args.length()+2;++i)vals.infallibleAppend(args.base()[i]);// Pad missing arguments with |undefined|.while(vals.length()<numFormals+1)vals.infallibleAppend(UndefinedValue());if(data.constructing)vals.infallibleAppend(args.newTarget());MOZ_ASSERT(vals.length()>=numFormals+1+data.constructing);data.maxArgv=vals.begin();}}else{data.constructing=false;data.numActualArgs=0;data.maxArgc=0;data.maxArgv=nullptr;data.envChain=state.asExecute()->environmentChain();data.calleeToken=CalleeToToken(state.script());if(state.script()->isForEval()&&state.script()->isDirectEvalInFunction()){// Push newTarget onto the stack.if(!vals.reserve(1))returnfalse;data.maxArgc=1;data.maxArgv=vals.begin();if(state.asExecute()->newTarget().isNull()){ScriptFrameIteriter(cx);vals.infallibleAppend(iter.newTarget());}else{vals.infallibleAppend(state.asExecute()->newTarget());}}}returntrue;}JitExecStatusjit::IonCannon(JSContext*cx,RunState&state){IonScript*ion=state.script()->ionScript();EnterJitDatadata(cx);data.jitcode=ion->method()->raw();Rooted<GCVector<Value>>vals(cx,GCVector<Value>(cx));if(!SetEnterJitData(cx,data,state,&vals))returnJitExec_Error;JitExecStatusstatus=EnterIon(cx,data);if(status==JitExec_Ok)state.setReturnValue(data.result);returnstatus;}JitExecStatusjit::FastInvoke(JSContext*cx,HandleFunctionfun,CallArgs&args){if(!CheckRecursionLimit(cx))returnJitExec_Error;RootedScriptscript(cx,fun->nonLazyScript());if(!Debugger::checkNoExecute(cx,script))returnJitExec_Error;#ifdef DEBUG// See comment in EnterBaseline.mozilla::Maybe<JS::AutoAssertNoGC>nogc;nogc.emplace(cx);#endifIonScript*ion=script->ionScript();JitCode*code=ion->method();void*jitcode=code->raw();MOZ_ASSERT(jit::IsIonEnabled(cx));MOZ_ASSERT(!ion->bailoutExpected());ActivationEntryMonitorentryMonitor(cx,CalleeToToken(script));JitActivationactivation(cx);EnterJitCodeenter=cx->runtime()->jitRuntime()->enterIon();void*calleeToken=CalleeToToken(fun,/* constructing = */false);RootedValueresult(cx,Int32Value(args.length()));MOZ_ASSERT(args.length()>=fun->nargs());#ifdef DEBUGnogc.reset();#endifCALL_GENERATED_CODE(enter,jitcode,args.length()+1,args.array()-1,/* osrFrame = */nullptr,calleeToken,/* envChain = */nullptr,0,result.address());MOZ_ASSERT(!cx->hasIonReturnOverride());args.rval().set(result);MOZ_ASSERT_IF(result.isMagic(),result.isMagic(JS_ION_ERROR));returnresult.isMagic()?JitExec_Error:JitExec_Ok;}staticvoidInvalidateActivation(FreeOp*fop,constJitActivationIterator&activations,boolinvalidateAll){JitSpew(JitSpew_IonInvalidate,"BEGIN invalidating activation");#ifdef CHECK_OSIPOINT_REGISTERSif(JitOptions.checkOsiPointRegisters)activations->asJit()->setCheckRegs(false);#endifsize_tframeno=1;for(JitFrameIteratorit(activations);!it.done();++it,++frameno){MOZ_ASSERT_IF(frameno==1,it.isExitFrame()||it.type()==JitFrame_Bailout);#ifdef JS_JITSPEWswitch(it.type()){caseJitFrame_Exit:JitSpew(JitSpew_IonInvalidate,"#%"PRIuSIZE" exit frame @ %p",frameno,it.fp());break;caseJitFrame_BaselineJS:caseJitFrame_IonJS:caseJitFrame_Bailout:{MOZ_ASSERT(it.isScripted());constchar*type="Unknown";if(it.isIonJS())type="Optimized";elseif(it.isBaselineJS())type="Baseline";elseif(it.isBailoutJS())type="Bailing";JitSpew(JitSpew_IonInvalidate,"#%"PRIuSIZE" %s JS frame @ %p, %s:%"PRIuSIZE" (fun: %p, script: %p, pc %p)",frameno,type,it.fp(),it.script()->maybeForwardedFilename(),it.script()->lineno(),it.maybeCallee(),(JSScript*)it.script(),it.returnAddressToFp());break;}caseJitFrame_BaselineStub:JitSpew(JitSpew_IonInvalidate,"#%"PRIuSIZE" baseline stub frame @ %p",frameno,it.fp());break;caseJitFrame_Rectifier:JitSpew(JitSpew_IonInvalidate,"#%"PRIuSIZE" rectifier frame @ %p",frameno,it.fp());break;caseJitFrame_IonICCall:JitSpew(JitSpew_IonInvalidate,"#%"PRIuSIZE" ion IC call frame @ %p",frameno,it.fp());break;caseJitFrame_Entry:JitSpew(JitSpew_IonInvalidate,"#%"PRIuSIZE" entry frame @ %p",frameno,it.fp());break;}#endif // JS_JITSPEWif(!it.isIonScripted())continue;boolcalledFromLinkStub=false;JitCode*lazyLinkStub=fop->runtime()->jitRuntime()->lazyLinkStub();if(it.returnAddressToFp()>=lazyLinkStub->raw()&&it.returnAddressToFp()<lazyLinkStub->rawEnd()){calledFromLinkStub=true;}// See if the frame has already been invalidated.if(!calledFromLinkStub&&it.checkInvalidation())continue;JSScript*script=it.script();if(!script->hasIonScript())continue;if(!invalidateAll&&!script->ionScript()->invalidated())continue;IonScript*ionScript=script->ionScript();// Purge ICs before we mark this script as invalidated. This will// prevent lastJump_ from appearing to be a bogus pointer, just// in case anyone tries to read it.ionScript->purgeICs(script->zone());ionScript->purgeOptimizedStubs(script->zone());// Clean up any pointers from elsewhere in the runtime to this IonScript// which is about to become disconnected from its JSScript.ionScript->unlinkFromRuntime(fop);// This frame needs to be invalidated. We do the following://// 1. Increment the reference counter to keep the ionScript alive// for the invalidation bailout or for the exception handler.// 2. Determine safepoint that corresponds to the current call.// 3. From safepoint, get distance to the OSI-patchable offset.// 4. From the IonScript, determine the distance between the// call-patchable offset and the invalidation epilogue.// 5. Patch the OSI point with a call-relative to the// invalidation epilogue.//// The code generator ensures that there's enough space for us// to patch in a call-relative operation at each invalidation// point.//// Note: you can't simplify this mechanism to "just patch the// instruction immediately after the call" because things may// need to move into a well-defined register state (using move// instructions after the call) in to capture an appropriate// snapshot after the call occurs.ionScript->incrementInvalidationCount();JitCode*ionCode=ionScript->method();JS::Zone*zone=script->zone();if(zone->needsIncrementalBarrier()){// We're about to remove edges from the JSScript to gcthings// embedded in the JitCode. Perform one final trace of the// JitCode for the incremental GC, as it must know about// those edges.ionCode->traceChildren(zone->barrierTracer());}ionCode->setInvalidated();// Don't adjust OSI points in the linkStub (which don't exist), or in a// bailout path.if(calledFromLinkStub||it.isBailoutJS())continue;// Write the delta (from the return address offset to the// IonScript pointer embedded into the invalidation epilogue)// where the safepointed call instruction used to be. We rely on// the call sequence causing the safepoint being >= the size of// a uint32, which is checked during safepoint index// construction.AutoWritableJitCodeawjc(ionCode);constSafepointIndex*si=ionScript->getSafepointIndex(it.returnAddressToFp());CodeLocationLabeldataLabelToMunge(it.returnAddressToFp());ptrdiff_tdelta=ionScript->invalidateEpilogueDataOffset()-(it.returnAddressToFp()-ionCode->raw());Assembler::PatchWrite_Imm32(dataLabelToMunge,Imm32(delta));CodeLocationLabelosiPatchPoint=SafepointReader::InvalidationPatchPoint(ionScript,si);CodeLocationLabelinvalidateEpilogue(ionCode,CodeOffset(ionScript->invalidateEpilogueOffset()));JitSpew(JitSpew_IonInvalidate," ! Invalidate ionScript %p (inv count %"PRIuSIZE") -> patching osipoint %p",ionScript,ionScript->invalidationCount(),(void*)osiPatchPoint.raw());Assembler::PatchWrite_NearCall(osiPatchPoint,invalidateEpilogue);}JitSpew(JitSpew_IonInvalidate,"END invalidating activation");}voidjit::InvalidateAll(FreeOp*fop,Zone*zone){// The caller should previously have cancelled off thread compilation.#ifdef DEBUGfor(CompartmentsInZoneItercomp(zone);!comp.done();comp.next())MOZ_ASSERT(!HasOffThreadIonCompile(comp));#endifif(zone->isAtomsZone())return;JSContext*cx=TlsContext.get();for(JitActivationIteratoriter(cx,zone->group()->ownerContext());!iter.done();++iter){if(iter->compartment()->zone()==zone){JitSpew(JitSpew_IonInvalidate,"Invalidating all frames for GC");InvalidateActivation(fop,iter,true);}}}voidjit::Invalidate(TypeZone&types,FreeOp*fop,constRecompileInfoVector&invalid,boolresetUses,boolcancelOffThread){JitSpew(JitSpew_IonInvalidate,"Start invalidation.");// Add an invalidation reference to all invalidated IonScripts to indicate// to the traversal which frames have been invalidated.size_tnumInvalidations=0;for(size_ti=0;i<invalid.length();i++){constCompilerOutput*co=invalid[i].compilerOutput(types);if(!co)continue;MOZ_ASSERT(co->isValid());if(cancelOffThread)CancelOffThreadIonCompile(co->script());if(!co->ion())continue;JitSpew(JitSpew_IonInvalidate," Invalidate %s:%"PRIuSIZE", IonScript %p",co->script()->filename(),co->script()->lineno(),co->ion());MarkJitProfilerEvent(fop->runtime(),co->script(),"Invalidate");// Keep the ion script alive during the invalidation and flag this// ionScript as being invalidated. This increment is removed by the// loop after the calls to InvalidateActivation.co->ion()->incrementInvalidationCount();numInvalidations++;}if(!numInvalidations){JitSpew(JitSpew_IonInvalidate," No IonScript invalidation.");return;}// This method can be called both during GC and during the course of normal// script execution. In the former case this class will already be on the// stack, and in the latter case the invalidations will all be on the// current thread's stack, but the assertion under ActivationIterator can't// tell that this is a thread local use of the iterator.JSRuntime::AutoProhibitActiveContextChangeapacc(fop->runtime());JSContext*cx=TlsContext.get();for(JitActivationIteratoriter(cx,types.zone()->group()->ownerContext());!iter.done();++iter)InvalidateActivation(fop,iter,false);// Drop the references added above. If a script was never active, its// IonScript will be immediately destroyed. Otherwise, it will be held live// until its last invalidated frame is destroyed.for(size_ti=0;i<invalid.length();i++){CompilerOutput*co=invalid[i].compilerOutput(types);if(!co)continue;MOZ_ASSERT(co->isValid());JSScript*script=co->script();IonScript*ionScript=co->ion();if(!ionScript)continue;script->setIonScript(nullptr,nullptr);ionScript->decrementInvalidationCount(fop);co->invalidate();numInvalidations--;// Wait for the scripts to get warm again before doing another// compile, unless we are recompiling *because* a script got hot// (resetUses is false).if(resetUses)script->resetWarmUpCounter();}// Make sure we didn't leak references by invalidating the same IonScript// multiple times in the above loop.MOZ_ASSERT(!numInvalidations);}voidjit::Invalidate(JSContext*cx,constRecompileInfoVector&invalid,boolresetUses,boolcancelOffThread){jit::Invalidate(cx->zone()->types,cx->runtime()->defaultFreeOp(),invalid,resetUses,cancelOffThread);}voidjit::IonScript::invalidate(JSContext*cx,boolresetUses,constchar*reason){JitSpew(JitSpew_IonInvalidate," Invalidate IonScript %p: %s",this,reason);// RecompileInfoVector has inline space for at least one element.RecompileInfoVectorlist;MOZ_RELEASE_ASSERT(list.reserve(1));list.infallibleAppend(recompileInfo());Invalidate(cx,list,resetUses,true);}voidjit::Invalidate(JSContext*cx,JSScript*script,boolresetUses,boolcancelOffThread){MOZ_ASSERT(script->hasIonScript());// RecompileInfoVector has inline space for at least one element.RecompileInfoVectorscripts;MOZ_ASSERT(script->hasIonScript());MOZ_RELEASE_ASSERT(scripts.reserve(1));scripts.infallibleAppend(script->ionScript()->recompileInfo());Invalidate(cx,scripts,resetUses,cancelOffThread);}staticvoidFinishInvalidationOf(FreeOp*fop,JSScript*script,IonScript*ionScript,booladdMarker){TypeZone&types=script->zone()->types;// Note: If the script is about to be swept, the compiler output may have// already been destroyed.if(CompilerOutput*output=ionScript->recompileInfo().compilerOutput(types))output->invalidate();// If this script has Ion code on the stack, invalidated() will return// true. In this case we have to wait until destroying it.if(!ionScript->invalidated())jit::IonScript::Destroy(fop,ionScript);// Register invalidation with profiler, if appropriate.if(addMarker)MarkJitProfilerEvent(fop->runtime(),script,"Invalidate (GC)");}voidjit::FinishInvalidation(FreeOp*fop,JSScript*script,booladdMarker){// In all cases, nullptr out script->ion to avoid re-entry.if(script->hasIonScript()){IonScript*ion=script->ionScript();script->setIonScript(nullptr,nullptr);FinishInvalidationOf(fop,script,ion,addMarker);}}voidjit::ForbidCompilation(JSContext*cx,JSScript*script){JitSpew(JitSpew_IonAbort,"Disabling Ion compilation of script %s:%"PRIuSIZE,script->filename(),script->lineno());CancelOffThreadIonCompile(script);if(script->hasIonScript())Invalidate(cx,script,false);script->setIonScript(cx->runtime(),ION_DISABLED_SCRIPT);MarkJitProfilerEvent(cx->runtime(),script,"Ion compilation disabled");}AutoFlushICache*JSContext::autoFlushICache()const{returnautoFlushICache_;}voidJSContext::setAutoFlushICache(AutoFlushICache*afc){autoFlushICache_=afc;}// Set the range for the merging of flushes. The flushing is deferred until the end of// the AutoFlushICache context. Subsequent flushing within this range will is also// deferred. This is only expected to be defined once for each AutoFlushICache// context. It assumes the range will be flushed is required to be within an// AutoFlushICache context.voidAutoFlushICache::setRange(uintptr_tstart,size_tlen){#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)AutoFlushICache*afc=TlsContext.get()->autoFlushICache();MOZ_ASSERT(afc);MOZ_ASSERT(!afc->start_);JitSpewCont(JitSpew_CacheFlush,"(%"PRIxPTR" %"PRIxSIZE"):",start,len);uintptr_tstop=start+len;afc->start_=start;afc->stop_=stop;#endif}// Flush the instruction cache.//// If called within a dynamic AutoFlushICache context and if the range is already pending// flushing for this AutoFlushICache context then the request is ignored with the// understanding that it will be flushed on exit from the AutoFlushICache context.// Otherwise the range is flushed immediately.//// Updates outside the current code object are typically the exception so they are flushed// immediately rather than attempting to merge them.//// For efficiency it is expected that all large ranges will be flushed within an// AutoFlushICache, so check. If this assertion is hit then it does not necessarily// indicate a program fault but it might indicate a lost opportunity to merge cache// flushing. It can be corrected by wrapping the call in an AutoFlushICache to context.//// Note this can be called without TLS JSContext defined so this case needs// to be guarded against. E.g. when patching instructions from the exception// handler on MacOS running the ARM simulator.voidAutoFlushICache::flush(uintptr_tstart,size_tlen){#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)JSContext*cx=TlsContext.get();AutoFlushICache*afc=cx?cx->autoFlushICache():nullptr;if(!afc){JitSpewCont(JitSpew_CacheFlush,"#");ExecutableAllocator::cacheFlush((void*)start,len);MOZ_ASSERT(len<=32);return;}uintptr_tstop=start+len;if(start>=afc->start_&&stop<=afc->stop_){// Update is within the pending flush range, so defer to the end of the context.JitSpewCont(JitSpew_CacheFlush,afc->inhibit_?"-":"=");return;}JitSpewCont(JitSpew_CacheFlush,afc->inhibit_?"x":"*");ExecutableAllocator::cacheFlush((void*)start,len);#endif}// Flag the current dynamic AutoFlushICache as inhibiting flushing. Useful in error paths// where the changes are being abandoned.voidAutoFlushICache::setInhibit(){#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)AutoFlushICache*afc=TlsContext.get()->autoFlushICache();MOZ_ASSERT(afc);MOZ_ASSERT(afc->start_);JitSpewCont(JitSpew_CacheFlush,"I");afc->inhibit_=true;#endif}// The common use case is merging cache flushes when preparing a code object. In this// case the entire range of the code object is being flushed and as the code is patched// smaller redundant flushes could occur. The design allows an AutoFlushICache dynamic// thread local context to be declared in which the range of the code object can be set// which defers flushing until the end of this dynamic context. The redundant flushing// within this code range is also deferred avoiding redundant flushing. Flushing outside// this code range is not affected and proceeds immediately.//// In some cases flushing is not necessary, such as when compiling an wasm module which// is flushed again when dynamically linked, and also in error paths that abandon the// code. Flushing within the set code range can be inhibited within the AutoFlushICache// dynamic context by setting an inhibit flag.//// The JS compiler can be re-entered while within an AutoFlushICache dynamic context and// it is assumed that code being assembled or patched is not executed before the exit of// the respective AutoFlushICache dynamic context.//AutoFlushICache::AutoFlushICache(constchar*nonce,boolinhibit)#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64):start_(0),stop_(0),name_(nonce),inhibit_(inhibit)#endif{#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)JSContext*cx=TlsContext.get();AutoFlushICache*afc=cx->autoFlushICache();if(afc)JitSpew(JitSpew_CacheFlush,"<%s,%s%s ",nonce,afc->name_,inhibit?" I":"");elseJitSpewCont(JitSpew_CacheFlush,"<%s%s ",nonce,inhibit?" I":"");prev_=afc;cx->setAutoFlushICache(this);#endif}AutoFlushICache::~AutoFlushICache(){#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)JSContext*cx=TlsContext.get();MOZ_ASSERT(cx->autoFlushICache()==this);if(!inhibit_&&start_)ExecutableAllocator::cacheFlush((void*)start_,size_t(stop_-start_));JitSpewCont(JitSpew_CacheFlush,"%s%s>",name_,start_?"":" U");JitSpewFin(JitSpew_CacheFlush);cx->setAutoFlushICache(prev_);#endif}size_tjit::SizeOfIonData(JSScript*script,mozilla::MallocSizeOfmallocSizeOf){size_tresult=0;if(script->hasIonScript())result+=script->ionScript()->sizeOfIncludingThis(mallocSizeOf);returnresult;}voidjit::DestroyJitScripts(FreeOp*fop,JSScript*script){if(script->hasIonScript())jit::IonScript::Destroy(fop,script->ionScript());if(script->hasBaselineScript())jit::BaselineScript::Destroy(fop,script->baselineScript());}voidjit::TraceJitScripts(JSTracer*trc,JSScript*script){if(script->hasIonScript())jit::IonScript::Trace(trc,script->ionScript());if(script->hasBaselineScript())jit::BaselineScript::Trace(trc,script->baselineScript());}booljit::JitSupportsFloatingPoint(){returnjs::jit::MacroAssembler::SupportsFloatingPoint();}booljit::JitSupportsUnalignedAccesses(){returnjs::jit::MacroAssembler::SupportsUnalignedAccesses();}booljit::JitSupportsSimd(){returnjs::jit::MacroAssembler::SupportsSimd();}booljit::JitSupportsAtomics(){#if defined(JS_CODEGEN_ARM)// Bug 1146902, bug 1077318: Enable Ion inlining of Atomics// operations on ARM only when the CPU has byte, halfword, and// doubleword load-exclusive and store-exclusive instructions,// until we can add support for systems that don't have those.returnjs::jit::HasLDSTREXBHD();#elsereturntrue;#endif}// If you change these, please also change the comment in TempAllocator./* static */constsize_tTempAllocator::BallastSize=16*1024;/* static */constsize_tTempAllocator::PreferredLifoChunkSize=32*1024;